From 5f8de423f190bbb79a62f804151bc24824fa32d8 Mon Sep 17 00:00:00 2001 From: "Matt A. Tobin" Date: Fri, 2 Feb 2018 04:16:08 -0500 Subject: Add m-esr52 at 52.6.0 --- netwerk/cache2/CacheStorageService.cpp | 2290 ++++++++++++++++++++++++++++++++ 1 file changed, 2290 insertions(+) create mode 100644 netwerk/cache2/CacheStorageService.cpp (limited to 'netwerk/cache2/CacheStorageService.cpp') diff --git a/netwerk/cache2/CacheStorageService.cpp b/netwerk/cache2/CacheStorageService.cpp new file mode 100644 index 000000000..67ef4c526 --- /dev/null +++ b/netwerk/cache2/CacheStorageService.cpp @@ -0,0 +1,2290 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "CacheLog.h" +#include "CacheStorageService.h" +#include "CacheFileIOManager.h" +#include "CacheObserver.h" +#include "CacheIndex.h" +#include "CacheIndexIterator.h" +#include "CacheStorage.h" +#include "AppCacheStorage.h" +#include "CacheEntry.h" +#include "CacheFileUtils.h" + +#include "OldWrappers.h" +#include "nsCacheService.h" +#include "nsDeleteDir.h" + +#include "nsICacheStorageVisitor.h" +#include "nsIObserverService.h" +#include "nsIFile.h" +#include "nsIURI.h" +#include "nsCOMPtr.h" +#include "nsAutoPtr.h" +#include "nsNetCID.h" +#include "nsNetUtil.h" +#include "nsServiceManagerUtils.h" +#include "nsWeakReference.h" +#include "mozilla/TimeStamp.h" +#include "mozilla/DebugOnly.h" +#include "mozilla/Services.h" + +namespace mozilla { +namespace net { + +namespace { + +void AppendMemoryStorageID(nsAutoCString &key) +{ + key.Append('/'); + key.Append('M'); +} + +} // namespace + +// Not defining as static or class member of CacheStorageService since +// it would otherwise need to include CacheEntry.h and that then would +// need to be exported to make nsNetModule.cpp compilable. +typedef nsClassHashtable + GlobalEntryTables; + +/** + * Keeps tables of entries. There is one entries table for each distinct load + * context type. The distinction is based on following load context info states: + * which builds a mapping key. + * + * Thread-safe to access, protected by the service mutex. + */ +static GlobalEntryTables* sGlobalEntryTables; + +CacheMemoryConsumer::CacheMemoryConsumer(uint32_t aFlags) +: mReportedMemoryConsumption(0) +, mFlags(aFlags) +{ +} + +void +CacheMemoryConsumer::DoMemoryReport(uint32_t aCurrentSize) +{ + if (!(mFlags & DONT_REPORT) && CacheStorageService::Self()) { + CacheStorageService::Self()->OnMemoryConsumptionChange(this, aCurrentSize); + } +} + +CacheStorageService::MemoryPool::MemoryPool(EType aType) +: mType(aType) +, mMemorySize(0) +{ +} + +CacheStorageService::MemoryPool::~MemoryPool() +{ + if (mMemorySize != 0) { + NS_ERROR("Network cache reported memory consumption is not at 0, probably leaking?"); + } +} + +uint32_t +CacheStorageService::MemoryPool::Limit() const +{ + switch (mType) { + case DISK: + return CacheObserver::MetadataMemoryLimit(); + case MEMORY: + return CacheObserver::MemoryCacheCapacity(); + } + + MOZ_CRASH("Bad pool type"); + return 0; +} + +NS_IMPL_ISUPPORTS(CacheStorageService, + nsICacheStorageService, + nsIMemoryReporter, + nsITimerCallback, + nsICacheTesting) + +CacheStorageService* CacheStorageService::sSelf = nullptr; + +CacheStorageService::CacheStorageService() +: mLock("CacheStorageService.mLock") +, mForcedValidEntriesLock("CacheStorageService.mForcedValidEntriesLock") +, mShutdown(false) +, mDiskPool(MemoryPool::DISK) +, mMemoryPool(MemoryPool::MEMORY) +{ + CacheFileIOManager::Init(); + + MOZ_ASSERT(!sSelf); + + sSelf = this; + sGlobalEntryTables = new GlobalEntryTables(); + + RegisterStrongMemoryReporter(this); +} + +CacheStorageService::~CacheStorageService() +{ + LOG(("CacheStorageService::~CacheStorageService")); + sSelf = nullptr; +} + +void CacheStorageService::Shutdown() +{ + mozilla::MutexAutoLock lock(mLock); + + if (mShutdown) + return; + + LOG(("CacheStorageService::Shutdown - start")); + + mShutdown = true; + + nsCOMPtr event = + NewRunnableMethod(this, &CacheStorageService::ShutdownBackground); + Dispatch(event); + +#ifdef NS_FREE_PERMANENT_DATA + sGlobalEntryTables->Clear(); + delete sGlobalEntryTables; +#endif + sGlobalEntryTables = nullptr; + + LOG(("CacheStorageService::Shutdown - done")); +} + +void CacheStorageService::ShutdownBackground() +{ + LOG(("CacheStorageService::ShutdownBackground - start")); + + MOZ_ASSERT(IsOnManagementThread()); + + { + mozilla::MutexAutoLock lock(mLock); + + // Cancel purge timer to avoid leaking. + if (mPurgeTimer) { + LOG((" freeing the timer")); + mPurgeTimer->Cancel(); + } + } + +#ifdef NS_FREE_PERMANENT_DATA + Pool(false).mFrecencyArray.Clear(); + Pool(false).mExpirationArray.Clear(); + Pool(true).mFrecencyArray.Clear(); + Pool(true).mExpirationArray.Clear(); +#endif + + LOG(("CacheStorageService::ShutdownBackground - done")); +} + +// Internal management methods + +namespace { + +// WalkCacheRunnable +// Base class for particular storage entries visiting +class WalkCacheRunnable : public Runnable + , public CacheStorageService::EntryInfoCallback +{ +protected: + WalkCacheRunnable(nsICacheStorageVisitor* aVisitor, + bool aVisitEntries) + : mService(CacheStorageService::Self()) + , mCallback(aVisitor) + , mSize(0) + , mNotifyStorage(true) + , mVisitEntries(aVisitEntries) + , mCancel(false) + { + MOZ_ASSERT(NS_IsMainThread()); + } + + virtual ~WalkCacheRunnable() + { + if (mCallback) { + ProxyReleaseMainThread(mCallback); + } + } + + RefPtr mService; + nsCOMPtr mCallback; + + uint64_t mSize; + + bool mNotifyStorage : 1; + bool mVisitEntries : 1; + + Atomic mCancel; +}; + +// WalkMemoryCacheRunnable +// Responsible to visit memory storage and walk +// all entries on it asynchronously. +class WalkMemoryCacheRunnable : public WalkCacheRunnable +{ +public: + WalkMemoryCacheRunnable(nsILoadContextInfo *aLoadInfo, + bool aVisitEntries, + nsICacheStorageVisitor* aVisitor) + : WalkCacheRunnable(aVisitor, aVisitEntries) + { + CacheFileUtils::AppendKeyPrefix(aLoadInfo, mContextKey); + MOZ_ASSERT(NS_IsMainThread()); + } + + nsresult Walk() + { + return mService->Dispatch(this); + } + +private: + NS_IMETHOD Run() override + { + if (CacheStorageService::IsOnManagementThread()) { + LOG(("WalkMemoryCacheRunnable::Run - collecting [this=%p]", this)); + // First, walk, count and grab all entries from the storage + + mozilla::MutexAutoLock lock(CacheStorageService::Self()->Lock()); + + if (!CacheStorageService::IsRunning()) + return NS_ERROR_NOT_INITIALIZED; + + CacheEntryTable* entries; + if (sGlobalEntryTables->Get(mContextKey, &entries)) { + for (auto iter = entries->Iter(); !iter.Done(); iter.Next()) { + CacheEntry* entry = iter.UserData(); + + // Ignore disk entries + if (entry->IsUsingDisk()) { + continue; + } + + mSize += entry->GetMetadataMemoryConsumption(); + + int64_t size; + if (NS_SUCCEEDED(entry->GetDataSize(&size))) { + mSize += size; + } + mEntryArray.AppendElement(entry); + } + } + + // Next, we dispatch to the main thread + } else if (NS_IsMainThread()) { + LOG(("WalkMemoryCacheRunnable::Run - notifying [this=%p]", this)); + + if (mNotifyStorage) { + LOG((" storage")); + + // Second, notify overall storage info + mCallback->OnCacheStorageInfo(mEntryArray.Length(), mSize, + CacheObserver::MemoryCacheCapacity(), nullptr); + if (!mVisitEntries) + return NS_OK; // done + + mNotifyStorage = false; + + } else { + LOG((" entry [left=%d, canceled=%d]", mEntryArray.Length(), (bool)mCancel)); + + // Third, notify each entry until depleted or canceled + if (!mEntryArray.Length() || mCancel) { + mCallback->OnCacheEntryVisitCompleted(); + return NS_OK; // done + } + + // Grab the next entry + RefPtr entry = mEntryArray[0]; + mEntryArray.RemoveElementAt(0); + + // Invokes this->OnEntryInfo, that calls the callback with all + // information of the entry. + CacheStorageService::GetCacheEntryInfo(entry, this); + } + } else { + MOZ_CRASH("Bad thread"); + return NS_ERROR_FAILURE; + } + + NS_DispatchToMainThread(this); + return NS_OK; + } + + virtual ~WalkMemoryCacheRunnable() + { + if (mCallback) + ProxyReleaseMainThread(mCallback); + } + + virtual void OnEntryInfo(const nsACString & aURISpec, const nsACString & aIdEnhance, + int64_t aDataSize, int32_t aFetchCount, + uint32_t aLastModifiedTime, uint32_t aExpirationTime, + bool aPinned) override + { + nsresult rv; + + nsCOMPtr uri; + rv = NS_NewURI(getter_AddRefs(uri), aURISpec); + if (NS_FAILED(rv)) { + return; + } + + rv = mCallback->OnCacheEntryInfo(uri, aIdEnhance, aDataSize, aFetchCount, + aLastModifiedTime, aExpirationTime, aPinned); + if (NS_FAILED(rv)) { + LOG((" callback failed, canceling the walk")); + mCancel = true; + } + } + +private: + nsCString mContextKey; + nsTArray > mEntryArray; +}; + +// WalkDiskCacheRunnable +// Using the cache index information to get the list of files per context. +class WalkDiskCacheRunnable : public WalkCacheRunnable +{ +public: + WalkDiskCacheRunnable(nsILoadContextInfo *aLoadInfo, + bool aVisitEntries, + nsICacheStorageVisitor* aVisitor) + : WalkCacheRunnable(aVisitor, aVisitEntries) + , mLoadInfo(aLoadInfo) + , mPass(COLLECT_STATS) + { + } + + nsresult Walk() + { + // TODO, bug 998693 + // Initial index build should be forced here so that about:cache soon + // after startup gives some meaningfull results. + + // Dispatch to the INDEX level in hope that very recent cache entries + // information gets to the index list before we grab the index iterator + // for the first time. This tries to avoid miss of entries that has + // been created right before the visit is required. + RefPtr thread = CacheFileIOManager::IOThread(); + NS_ENSURE_TRUE(thread, NS_ERROR_NOT_INITIALIZED); + + return thread->Dispatch(this, CacheIOThread::INDEX); + } + +private: + // Invokes OnCacheEntryInfo callback for each single found entry. + // There is one instance of this class per one entry. + class OnCacheEntryInfoRunnable : public Runnable + { + public: + explicit OnCacheEntryInfoRunnable(WalkDiskCacheRunnable* aWalker) + : mWalker(aWalker) + { + } + + NS_IMETHOD Run() override + { + MOZ_ASSERT(NS_IsMainThread()); + + nsresult rv; + + nsCOMPtr uri; + rv = NS_NewURI(getter_AddRefs(uri), mURISpec); + if (NS_FAILED(rv)) { + return NS_OK; + } + + rv = mWalker->mCallback->OnCacheEntryInfo( + uri, mIdEnhance, mDataSize, mFetchCount, + mLastModifiedTime, mExpirationTime, mPinned); + if (NS_FAILED(rv)) { + mWalker->mCancel = true; + } + + return NS_OK; + } + + RefPtr mWalker; + + nsCString mURISpec; + nsCString mIdEnhance; + int64_t mDataSize; + int32_t mFetchCount; + uint32_t mLastModifiedTime; + uint32_t mExpirationTime; + bool mPinned; + }; + + NS_IMETHOD Run() override + { + // The main loop + nsresult rv; + + if (CacheStorageService::IsOnManagementThread()) { + switch (mPass) { + case COLLECT_STATS: + // Get quickly the cache stats. + uint32_t size; + rv = CacheIndex::GetCacheStats(mLoadInfo, &size, &mCount); + if (NS_FAILED(rv)) { + if (mVisitEntries) { + // both onStorageInfo and onCompleted are expected + NS_DispatchToMainThread(this); + } + return NS_DispatchToMainThread(this); + } + + mSize = size << 10; + + // Invoke onCacheStorageInfo with valid information. + NS_DispatchToMainThread(this); + + if (!mVisitEntries) { + return NS_OK; // done + } + + mPass = ITERATE_METADATA; + MOZ_FALLTHROUGH; + + case ITERATE_METADATA: + // Now grab the context iterator. + if (!mIter) { + rv = CacheIndex::GetIterator(mLoadInfo, true, getter_AddRefs(mIter)); + if (NS_FAILED(rv)) { + // Invoke onCacheEntryVisitCompleted now + return NS_DispatchToMainThread(this); + } + } + + while (!mCancel && !CacheObserver::ShuttingDown()) { + if (CacheIOThread::YieldAndRerun()) + return NS_OK; + + SHA1Sum::Hash hash; + rv = mIter->GetNextHash(&hash); + if (NS_FAILED(rv)) + break; // done (or error?) + + // This synchronously invokes OnEntryInfo on this class where we + // redispatch to the main thread for the consumer callback. + CacheFileIOManager::GetEntryInfo(&hash, this); + } + + // Invoke onCacheEntryVisitCompleted on the main thread + NS_DispatchToMainThread(this); + } + } else if (NS_IsMainThread()) { + if (mNotifyStorage) { + nsCOMPtr dir; + CacheFileIOManager::GetCacheDirectory(getter_AddRefs(dir)); + mCallback->OnCacheStorageInfo(mCount, mSize, CacheObserver::DiskCacheCapacity(), dir); + mNotifyStorage = false; + } else { + mCallback->OnCacheEntryVisitCompleted(); + } + } else { + MOZ_CRASH("Bad thread"); + return NS_ERROR_FAILURE; + } + + return NS_OK; + } + + virtual void OnEntryInfo(const nsACString & aURISpec, const nsACString & aIdEnhance, + int64_t aDataSize, int32_t aFetchCount, + uint32_t aLastModifiedTime, uint32_t aExpirationTime, + bool aPinned) override + { + // Called directly from CacheFileIOManager::GetEntryInfo. + + // Invoke onCacheEntryInfo on the main thread for this entry. + RefPtr info = new OnCacheEntryInfoRunnable(this); + info->mURISpec = aURISpec; + info->mIdEnhance = aIdEnhance; + info->mDataSize = aDataSize; + info->mFetchCount = aFetchCount; + info->mLastModifiedTime = aLastModifiedTime; + info->mExpirationTime = aExpirationTime; + info->mPinned = aPinned; + + NS_DispatchToMainThread(info); + } + + RefPtr mLoadInfo; + enum { + // First, we collect stats for the load context. + COLLECT_STATS, + + // Second, if demanded, we iterate over the entries gethered + // from the iterator and call CacheFileIOManager::GetEntryInfo + // for each found entry. + ITERATE_METADATA, + } mPass; + + RefPtr mIter; + uint32_t mCount; +}; + +} // namespace + +void CacheStorageService::DropPrivateBrowsingEntries() +{ + mozilla::MutexAutoLock lock(mLock); + + if (mShutdown) + return; + + nsTArray keys; + for (auto iter = sGlobalEntryTables->Iter(); !iter.Done(); iter.Next()) { + const nsACString& key = iter.Key(); + nsCOMPtr info = CacheFileUtils::ParseKey(key); + if (info && info->IsPrivate()) { + keys.AppendElement(key); + } + } + + for (uint32_t i = 0; i < keys.Length(); ++i) { + DoomStorageEntries(keys[i], nullptr, true, false, nullptr); + } +} + +namespace { + +class CleaupCacheDirectoriesRunnable : public Runnable +{ +public: + NS_DECL_NSIRUNNABLE + static bool Post(uint32_t aVersion, uint32_t aActive); + +private: + CleaupCacheDirectoriesRunnable(uint32_t aVersion, uint32_t aActive) + : mVersion(aVersion), mActive(aActive) + { + nsCacheService::GetDiskCacheDirectory(getter_AddRefs(mCache1Dir)); + CacheFileIOManager::GetCacheDirectory(getter_AddRefs(mCache2Dir)); +#if defined(MOZ_WIDGET_ANDROID) + CacheFileIOManager::GetProfilelessCacheDirectory(getter_AddRefs(mCache2Profileless)); +#endif + } + + virtual ~CleaupCacheDirectoriesRunnable() {} + uint32_t mVersion, mActive; + nsCOMPtr mCache1Dir, mCache2Dir; +#if defined(MOZ_WIDGET_ANDROID) + nsCOMPtr mCache2Profileless; +#endif +}; + +// static +bool CleaupCacheDirectoriesRunnable::Post(uint32_t aVersion, uint32_t aActive) +{ + // CleaupCacheDirectories is called regardless what cache version is set up to use. + // To obtain the cache1 directory we must unfortunatelly instantiate the old cache + // service despite it may not be used at all... This also initialize nsDeleteDir. + nsCOMPtr service = do_GetService(NS_CACHESERVICE_CONTRACTID); + if (!service) + return false; + + nsCOMPtr thread; + service->GetCacheIOTarget(getter_AddRefs(thread)); + if (!thread) + return false; + + RefPtr r = + new CleaupCacheDirectoriesRunnable(aVersion, aActive); + thread->Dispatch(r, NS_DISPATCH_NORMAL); + return true; +} + +NS_IMETHODIMP CleaupCacheDirectoriesRunnable::Run() +{ + MOZ_ASSERT(!NS_IsMainThread()); + + if (mCache1Dir) { + nsDeleteDir::RemoveOldTrashes(mCache1Dir); + } + if (mCache2Dir) { + nsDeleteDir::RemoveOldTrashes(mCache2Dir); + } +#if defined(MOZ_WIDGET_ANDROID) + if (mCache2Profileless) { + nsDeleteDir::RemoveOldTrashes(mCache2Profileless); + // Always delete the profileless cache on Android + nsDeleteDir::DeleteDir(mCache2Profileless, true, 30000); + } +#endif + + // Delete the non-active version cache data right now + if (mVersion == mActive) { + return NS_OK; + } + + switch (mVersion) { + case 0: + if (mCache1Dir) { + nsDeleteDir::DeleteDir(mCache1Dir, true, 30000); + } + break; + case 1: + if (mCache2Dir) { + nsDeleteDir::DeleteDir(mCache2Dir, true, 30000); + } + break; + } + + return NS_OK; +} + +} // namespace + +// static +void CacheStorageService::CleaupCacheDirectories(uint32_t aVersion, uint32_t aActive) +{ + // Make sure we schedule just once in case CleaupCacheDirectories gets called + // multiple times from some reason. + static bool runOnce = CleaupCacheDirectoriesRunnable::Post(aVersion, aActive); + if (!runOnce) { + NS_WARNING("Could not start cache trashes cleanup"); + } +} + +// Helper methods + +// static +bool CacheStorageService::IsOnManagementThread() +{ + RefPtr service = Self(); + if (!service) + return false; + + nsCOMPtr target = service->Thread(); + if (!target) + return false; + + bool currentThread; + nsresult rv = target->IsOnCurrentThread(¤tThread); + return NS_SUCCEEDED(rv) && currentThread; +} + +already_AddRefed CacheStorageService::Thread() const +{ + return CacheFileIOManager::IOTarget(); +} + +nsresult CacheStorageService::Dispatch(nsIRunnable* aEvent) +{ + RefPtr cacheIOThread = CacheFileIOManager::IOThread(); + if (!cacheIOThread) + return NS_ERROR_NOT_AVAILABLE; + + return cacheIOThread->Dispatch(aEvent, CacheIOThread::MANAGEMENT); +} + +// nsICacheStorageService + +NS_IMETHODIMP CacheStorageService::MemoryCacheStorage(nsILoadContextInfo *aLoadContextInfo, + nsICacheStorage * *_retval) +{ + NS_ENSURE_ARG(aLoadContextInfo); + NS_ENSURE_ARG(_retval); + + nsCOMPtr storage; + if (CacheObserver::UseNewCache()) { + storage = new CacheStorage(aLoadContextInfo, false, false, false, false); + } + else { + storage = new _OldStorage(aLoadContextInfo, false, false, false, nullptr); + } + + storage.forget(_retval); + return NS_OK; +} + +NS_IMETHODIMP CacheStorageService::DiskCacheStorage(nsILoadContextInfo *aLoadContextInfo, + bool aLookupAppCache, + nsICacheStorage * *_retval) +{ + NS_ENSURE_ARG(aLoadContextInfo); + NS_ENSURE_ARG(_retval); + + // TODO save some heap granularity - cache commonly used storages. + + // When disk cache is disabled, still provide a storage, but just keep stuff + // in memory. + bool useDisk = CacheObserver::UseDiskCache(); + + nsCOMPtr storage; + if (CacheObserver::UseNewCache()) { + storage = new CacheStorage(aLoadContextInfo, useDisk, aLookupAppCache, false /* size limit */, false /* don't pin */); + } + else { + storage = new _OldStorage(aLoadContextInfo, useDisk, aLookupAppCache, false, nullptr); + } + + storage.forget(_retval); + return NS_OK; +} + +NS_IMETHODIMP CacheStorageService::PinningCacheStorage(nsILoadContextInfo *aLoadContextInfo, + nsICacheStorage * *_retval) +{ + NS_ENSURE_ARG(aLoadContextInfo); + NS_ENSURE_ARG(_retval); + + if (!CacheObserver::UseNewCache()) { + return NS_ERROR_NOT_IMPLEMENTED; + } + + // When disk cache is disabled don't pretend we cache. + if (!CacheObserver::UseDiskCache()) { + return NS_ERROR_NOT_AVAILABLE; + } + + nsCOMPtr storage = new CacheStorage( + aLoadContextInfo, true /* use disk */, false /* no appcache */, true /* ignore size checks */, true /* pin */); + storage.forget(_retval); + return NS_OK; +} + +NS_IMETHODIMP CacheStorageService::AppCacheStorage(nsILoadContextInfo *aLoadContextInfo, + nsIApplicationCache *aApplicationCache, + nsICacheStorage * *_retval) +{ + NS_ENSURE_ARG(aLoadContextInfo); + NS_ENSURE_ARG(_retval); + + nsCOMPtr storage; + if (CacheObserver::UseNewCache()) { + // Using classification since cl believes we want to instantiate this method + // having the same name as the desired class... + storage = new mozilla::net::AppCacheStorage(aLoadContextInfo, aApplicationCache); + } + else { + storage = new _OldStorage(aLoadContextInfo, true, false, true, aApplicationCache); + } + + storage.forget(_retval); + return NS_OK; +} + +NS_IMETHODIMP CacheStorageService::SynthesizedCacheStorage(nsILoadContextInfo *aLoadContextInfo, + nsICacheStorage * *_retval) +{ + NS_ENSURE_ARG(aLoadContextInfo); + NS_ENSURE_ARG(_retval); + + nsCOMPtr storage; + if (CacheObserver::UseNewCache()) { + storage = new CacheStorage(aLoadContextInfo, false, false, true /* skip size checks for synthesized cache */, false /* no pinning */); + } + else { + storage = new _OldStorage(aLoadContextInfo, false, false, false, nullptr); + } + + storage.forget(_retval); + return NS_OK; +} + +NS_IMETHODIMP CacheStorageService::Clear() +{ + nsresult rv; + + if (CacheObserver::UseNewCache()) { + // Tell the index to block notification to AsyncGetDiskConsumption. + // Will be allowed again from CacheFileContextEvictor::EvictEntries() + // when all the context have been removed from disk. + CacheIndex::OnAsyncEviction(true); + + { + mozilla::MutexAutoLock lock(mLock); + + { + mozilla::MutexAutoLock forcedValidEntriesLock(mForcedValidEntriesLock); + mForcedValidEntries.Clear(); + } + + NS_ENSURE_TRUE(!mShutdown, NS_ERROR_NOT_INITIALIZED); + + nsTArray keys; + for (auto iter = sGlobalEntryTables->Iter(); !iter.Done(); iter.Next()) { + keys.AppendElement(iter.Key()); + } + + for (uint32_t i = 0; i < keys.Length(); ++i) { + DoomStorageEntries(keys[i], nullptr, true, false, nullptr); + } + + // Passing null as a load info means to evict all contexts. + // EvictByContext() respects the entry pinning. EvictAll() does not. + rv = CacheFileIOManager::EvictByContext(nullptr, false); + NS_ENSURE_SUCCESS(rv, rv); + } + } else { + nsCOMPtr serv = + do_GetService(NS_CACHESERVICE_CONTRACTID, &rv); + NS_ENSURE_SUCCESS(rv, rv); + + rv = serv->EvictEntries(nsICache::STORE_ANYWHERE); + NS_ENSURE_SUCCESS(rv, rv); + } + + return NS_OK; +} + +NS_IMETHODIMP CacheStorageService::PurgeFromMemory(uint32_t aWhat) +{ + uint32_t what; + + switch (aWhat) { + case PURGE_DISK_DATA_ONLY: + what = CacheEntry::PURGE_DATA_ONLY_DISK_BACKED; + break; + + case PURGE_DISK_ALL: + what = CacheEntry::PURGE_WHOLE_ONLY_DISK_BACKED; + break; + + case PURGE_EVERYTHING: + what = CacheEntry::PURGE_WHOLE; + break; + + default: + return NS_ERROR_INVALID_ARG; + } + + nsCOMPtr event = + new PurgeFromMemoryRunnable(this, what); + + return Dispatch(event); +} + +NS_IMETHODIMP CacheStorageService::PurgeFromMemoryRunnable::Run() +{ + if (NS_IsMainThread()) { + nsCOMPtr observerService = + mozilla::services::GetObserverService(); + if (observerService) { + observerService->NotifyObservers(nullptr, "cacheservice:purge-memory-pools", nullptr); + } + + return NS_OK; + } + + if (mService) { + // TODO not all flags apply to both pools + mService->Pool(true).PurgeAll(mWhat); + mService->Pool(false).PurgeAll(mWhat); + mService = nullptr; + } + + NS_DispatchToMainThread(this); + return NS_OK; +} + +NS_IMETHODIMP CacheStorageService::AsyncGetDiskConsumption( + nsICacheStorageConsumptionObserver* aObserver) +{ + NS_ENSURE_ARG(aObserver); + + nsresult rv; + + if (CacheObserver::UseNewCache()) { + rv = CacheIndex::AsyncGetDiskConsumption(aObserver); + NS_ENSURE_SUCCESS(rv, rv); + } else { + rv = _OldGetDiskConsumption::Get(aObserver); + NS_ENSURE_SUCCESS(rv, rv); + } + + return NS_OK; +} + +NS_IMETHODIMP CacheStorageService::GetIoTarget(nsIEventTarget** aEventTarget) +{ + NS_ENSURE_ARG(aEventTarget); + + if (CacheObserver::UseNewCache()) { + nsCOMPtr ioTarget = CacheFileIOManager::IOTarget(); + ioTarget.forget(aEventTarget); + } + else { + nsresult rv; + + nsCOMPtr serv = + do_GetService(NS_CACHESERVICE_CONTRACTID, &rv); + NS_ENSURE_SUCCESS(rv, rv); + + rv = serv->GetCacheIOTarget(aEventTarget); + NS_ENSURE_SUCCESS(rv, rv); + } + + return NS_OK; +} + +// Methods used by CacheEntry for management of in-memory structures. + +namespace { + +class FrecencyComparator +{ +public: + bool Equals(CacheEntry* a, CacheEntry* b) const { + return a->GetFrecency() == b->GetFrecency(); + } + bool LessThan(CacheEntry* a, CacheEntry* b) const { + return a->GetFrecency() < b->GetFrecency(); + } +}; + +class ExpirationComparator +{ +public: + bool Equals(CacheEntry* a, CacheEntry* b) const { + return a->GetExpirationTime() == b->GetExpirationTime(); + } + bool LessThan(CacheEntry* a, CacheEntry* b) const { + return a->GetExpirationTime() < b->GetExpirationTime(); + } +}; + +} // namespace + +void +CacheStorageService::RegisterEntry(CacheEntry* aEntry) +{ + MOZ_ASSERT(IsOnManagementThread()); + + if (mShutdown || !aEntry->CanRegister()) + return; + + TelemetryRecordEntryCreation(aEntry); + + LOG(("CacheStorageService::RegisterEntry [entry=%p]", aEntry)); + + MemoryPool& pool = Pool(aEntry->IsUsingDisk()); + pool.mFrecencyArray.AppendElement(aEntry); + pool.mExpirationArray.AppendElement(aEntry); + + aEntry->SetRegistered(true); +} + +void +CacheStorageService::UnregisterEntry(CacheEntry* aEntry) +{ + MOZ_ASSERT(IsOnManagementThread()); + + if (!aEntry->IsRegistered()) + return; + + TelemetryRecordEntryRemoval(aEntry); + + LOG(("CacheStorageService::UnregisterEntry [entry=%p]", aEntry)); + + MemoryPool& pool = Pool(aEntry->IsUsingDisk()); + mozilla::DebugOnly removedFrecency = pool.mFrecencyArray.RemoveElement(aEntry); + mozilla::DebugOnly removedExpiration = pool.mExpirationArray.RemoveElement(aEntry); + + MOZ_ASSERT(mShutdown || (removedFrecency && removedExpiration)); + + // Note: aEntry->CanRegister() since now returns false + aEntry->SetRegistered(false); +} + +static bool +AddExactEntry(CacheEntryTable* aEntries, + nsACString const& aKey, + CacheEntry* aEntry, + bool aOverwrite) +{ + RefPtr existingEntry; + if (!aOverwrite && aEntries->Get(aKey, getter_AddRefs(existingEntry))) { + bool equals = existingEntry == aEntry; + LOG(("AddExactEntry [entry=%p equals=%d]", aEntry, equals)); + return equals; // Already there... + } + + LOG(("AddExactEntry [entry=%p put]", aEntry)); + aEntries->Put(aKey, aEntry); + return true; +} + +static bool +RemoveExactEntry(CacheEntryTable* aEntries, + nsACString const& aKey, + CacheEntry* aEntry, + bool aOverwrite) +{ + RefPtr existingEntry; + if (!aEntries->Get(aKey, getter_AddRefs(existingEntry))) { + LOG(("RemoveExactEntry [entry=%p already gone]", aEntry)); + return false; // Already removed... + } + + if (!aOverwrite && existingEntry != aEntry) { + LOG(("RemoveExactEntry [entry=%p already replaced]", aEntry)); + return false; // Already replaced... + } + + LOG(("RemoveExactEntry [entry=%p removed]", aEntry)); + aEntries->Remove(aKey); + return true; +} + +bool +CacheStorageService::RemoveEntry(CacheEntry* aEntry, bool aOnlyUnreferenced) +{ + LOG(("CacheStorageService::RemoveEntry [entry=%p]", aEntry)); + + nsAutoCString entryKey; + nsresult rv = aEntry->HashingKey(entryKey); + if (NS_FAILED(rv)) { + NS_ERROR("aEntry->HashingKey() failed?"); + return false; + } + + mozilla::MutexAutoLock lock(mLock); + + if (mShutdown) { + LOG((" after shutdown")); + return false; + } + + if (aOnlyUnreferenced) { + if (aEntry->IsReferenced()) { + LOG((" still referenced, not removing")); + return false; + } + + if (!aEntry->IsUsingDisk() && IsForcedValidEntry(aEntry->GetStorageID(), entryKey)) { + LOG((" forced valid, not removing")); + return false; + } + } + + CacheEntryTable* entries; + if (sGlobalEntryTables->Get(aEntry->GetStorageID(), &entries)) + RemoveExactEntry(entries, entryKey, aEntry, false /* don't overwrite */); + + nsAutoCString memoryStorageID(aEntry->GetStorageID()); + AppendMemoryStorageID(memoryStorageID); + + if (sGlobalEntryTables->Get(memoryStorageID, &entries)) + RemoveExactEntry(entries, entryKey, aEntry, false /* don't overwrite */); + + return true; +} + +void +CacheStorageService::RecordMemoryOnlyEntry(CacheEntry* aEntry, + bool aOnlyInMemory, + bool aOverwrite) +{ + LOG(("CacheStorageService::RecordMemoryOnlyEntry [entry=%p, memory=%d, overwrite=%d]", + aEntry, aOnlyInMemory, aOverwrite)); + // This method is responsible to put this entry to a special record hashtable + // that contains only entries that are stored in memory. + // Keep in mind that every entry, regardless of whether is in-memory-only or not + // is always recorded in the storage master hash table, the one identified by + // CacheEntry.StorageID(). + + mLock.AssertCurrentThreadOwns(); + + if (mShutdown) { + LOG((" after shutdown")); + return; + } + + nsresult rv; + + nsAutoCString entryKey; + rv = aEntry->HashingKey(entryKey); + if (NS_FAILED(rv)) { + NS_ERROR("aEntry->HashingKey() failed?"); + return; + } + + CacheEntryTable* entries = nullptr; + nsAutoCString memoryStorageID(aEntry->GetStorageID()); + AppendMemoryStorageID(memoryStorageID); + + if (!sGlobalEntryTables->Get(memoryStorageID, &entries)) { + if (!aOnlyInMemory) { + LOG((" not recorded as memory only")); + return; + } + + entries = new CacheEntryTable(CacheEntryTable::MEMORY_ONLY); + sGlobalEntryTables->Put(memoryStorageID, entries); + LOG((" new memory-only storage table for %s", memoryStorageID.get())); + } + + if (aOnlyInMemory) { + AddExactEntry(entries, entryKey, aEntry, aOverwrite); + } + else { + RemoveExactEntry(entries, entryKey, aEntry, aOverwrite); + } +} + +// Checks if a cache entry is forced valid (will be loaded directly from cache +// without further validation) - see nsICacheEntry.idl for further details +bool CacheStorageService::IsForcedValidEntry(nsACString const &aContextKey, + nsACString const &aEntryKey) +{ + return IsForcedValidEntry(aContextKey + aEntryKey); +} + +bool CacheStorageService::IsForcedValidEntry(nsACString const &aContextEntryKey) +{ + mozilla::MutexAutoLock lock(mForcedValidEntriesLock); + + TimeStamp validUntil; + + if (!mForcedValidEntries.Get(aContextEntryKey, &validUntil)) { + return false; + } + + if (validUntil.IsNull()) { + return false; + } + + // Entry timeout not reached yet + if (TimeStamp::NowLoRes() <= validUntil) { + return true; + } + + // Entry timeout has been reached + mForcedValidEntries.Remove(aContextEntryKey); + return false; +} + +// Allows a cache entry to be loaded directly from cache without further +// validation - see nsICacheEntry.idl for further details +void CacheStorageService::ForceEntryValidFor(nsACString const &aContextKey, + nsACString const &aEntryKey, + uint32_t aSecondsToTheFuture) +{ + mozilla::MutexAutoLock lock(mForcedValidEntriesLock); + + TimeStamp now = TimeStamp::NowLoRes(); + ForcedValidEntriesPrune(now); + + // This will be the timeout + TimeStamp validUntil = now + TimeDuration::FromSeconds(aSecondsToTheFuture); + + mForcedValidEntries.Put(aContextKey + aEntryKey, validUntil); +} + +void CacheStorageService::RemoveEntryForceValid(nsACString const &aContextKey, + nsACString const &aEntryKey) +{ + mozilla::MutexAutoLock lock(mForcedValidEntriesLock); + + LOG(("CacheStorageService::RemoveEntryForceValid context='%s' entryKey=%s", + aContextKey.BeginReading(), aEntryKey.BeginReading())); + mForcedValidEntries.Remove(aContextKey + aEntryKey); +} + +// Cleans out the old entries in mForcedValidEntries +void CacheStorageService::ForcedValidEntriesPrune(TimeStamp &now) +{ + static TimeDuration const oneMinute = TimeDuration::FromSeconds(60); + static TimeStamp dontPruneUntil = now + oneMinute; + if (now < dontPruneUntil) + return; + + for (auto iter = mForcedValidEntries.Iter(); !iter.Done(); iter.Next()) { + if (iter.Data() < now) { + iter.Remove(); + } + } + dontPruneUntil = now + oneMinute; +} + +void +CacheStorageService::OnMemoryConsumptionChange(CacheMemoryConsumer* aConsumer, + uint32_t aCurrentMemoryConsumption) +{ + LOG(("CacheStorageService::OnMemoryConsumptionChange [consumer=%p, size=%u]", + aConsumer, aCurrentMemoryConsumption)); + + uint32_t savedMemorySize = aConsumer->mReportedMemoryConsumption; + if (savedMemorySize == aCurrentMemoryConsumption) + return; + + // Exchange saved size with current one. + aConsumer->mReportedMemoryConsumption = aCurrentMemoryConsumption; + + bool usingDisk = !(aConsumer->mFlags & CacheMemoryConsumer::MEMORY_ONLY); + bool overLimit = Pool(usingDisk).OnMemoryConsumptionChange( + savedMemorySize, aCurrentMemoryConsumption); + + if (!overLimit) + return; + + // It's likely the timer has already been set when we get here, + // check outside the lock to save resources. + if (mPurgeTimer) + return; + + // We don't know if this is called under the service lock or not, + // hence rather dispatch. + RefPtr cacheIOTarget = Thread(); + if (!cacheIOTarget) + return; + + // Dispatch as a priority task, we want to set the purge timer + // ASAP to prevent vain redispatch of this event. + nsCOMPtr event = + NewRunnableMethod(this, &CacheStorageService::SchedulePurgeOverMemoryLimit); + cacheIOTarget->Dispatch(event, nsIEventTarget::DISPATCH_NORMAL); +} + +bool +CacheStorageService::MemoryPool::OnMemoryConsumptionChange(uint32_t aSavedMemorySize, + uint32_t aCurrentMemoryConsumption) +{ + mMemorySize -= aSavedMemorySize; + mMemorySize += aCurrentMemoryConsumption; + + LOG((" mMemorySize=%u (+%u,-%u)", uint32_t(mMemorySize), aCurrentMemoryConsumption, aSavedMemorySize)); + + // Bypass purging when memory has not grew up significantly + if (aCurrentMemoryConsumption <= aSavedMemorySize) + return false; + + return mMemorySize > Limit(); +} + +void +CacheStorageService::SchedulePurgeOverMemoryLimit() +{ + LOG(("CacheStorageService::SchedulePurgeOverMemoryLimit")); + + mozilla::MutexAutoLock lock(mLock); + + if (mShutdown) { + LOG((" past shutdown")); + return; + } + + if (mPurgeTimer) { + LOG((" timer already up")); + return; + } + + mPurgeTimer = do_CreateInstance(NS_TIMER_CONTRACTID); + if (mPurgeTimer) { + nsresult rv; + rv = mPurgeTimer->InitWithCallback(this, 1000, nsITimer::TYPE_ONE_SHOT); + LOG((" timer init rv=0x%08x", rv)); + } +} + +NS_IMETHODIMP +CacheStorageService::Notify(nsITimer* aTimer) +{ + LOG(("CacheStorageService::Notify")); + + mozilla::MutexAutoLock lock(mLock); + + if (aTimer == mPurgeTimer) { + mPurgeTimer = nullptr; + + nsCOMPtr event = + NewRunnableMethod(this, &CacheStorageService::PurgeOverMemoryLimit); + Dispatch(event); + } + + return NS_OK; +} + +void +CacheStorageService::PurgeOverMemoryLimit() +{ + MOZ_ASSERT(IsOnManagementThread()); + + LOG(("CacheStorageService::PurgeOverMemoryLimit")); + + static TimeDuration const kFourSeconds = TimeDuration::FromSeconds(4); + TimeStamp now = TimeStamp::NowLoRes(); + + if (!mLastPurgeTime.IsNull() && now - mLastPurgeTime < kFourSeconds) { + LOG((" bypassed, too soon")); + return; + } + + mLastPurgeTime = now; + + Pool(true).PurgeOverMemoryLimit(); + Pool(false).PurgeOverMemoryLimit(); +} + +void +CacheStorageService::MemoryPool::PurgeOverMemoryLimit() +{ + TimeStamp start(TimeStamp::Now()); + + uint32_t const memoryLimit = Limit(); + if (mMemorySize > memoryLimit) { + LOG((" memory data consumption over the limit, abandon expired entries")); + PurgeExpired(); + } + + bool frecencyNeedsSort = true; + + // No longer makes sense since: + // Memory entries are never purged partially, only as a whole when the memory + // cache limit is overreached. + // Disk entries throw the data away ASAP so that only metadata are kept. + // TODO when this concept of two separate pools is found working, the code should + // clean up. +#if 0 + if (mMemorySize > memoryLimit) { + LOG((" memory data consumption over the limit, abandon disk backed data")); + PurgeByFrecency(frecencyNeedsSort, CacheEntry::PURGE_DATA_ONLY_DISK_BACKED); + } + + if (mMemorySize > memoryLimit) { + LOG((" metadata consumtion over the limit, abandon disk backed entries")); + PurgeByFrecency(frecencyNeedsSort, CacheEntry::PURGE_WHOLE_ONLY_DISK_BACKED); + } +#endif + + if (mMemorySize > memoryLimit) { + LOG((" memory data consumption over the limit, abandon any entry")); + PurgeByFrecency(frecencyNeedsSort, CacheEntry::PURGE_WHOLE); + } + + LOG((" purging took %1.2fms", (TimeStamp::Now() - start).ToMilliseconds())); +} + +void +CacheStorageService::MemoryPool::PurgeExpired() +{ + MOZ_ASSERT(IsOnManagementThread()); + + mExpirationArray.Sort(ExpirationComparator()); + uint32_t now = NowInSeconds(); + + uint32_t const memoryLimit = Limit(); + + for (uint32_t i = 0; mMemorySize > memoryLimit && i < mExpirationArray.Length();) { + if (CacheIOThread::YieldAndRerun()) + return; + + RefPtr entry = mExpirationArray[i]; + + uint32_t expirationTime = entry->GetExpirationTime(); + if (expirationTime > 0 && expirationTime <= now && + entry->Purge(CacheEntry::PURGE_WHOLE)) { + LOG((" purged expired, entry=%p, exptime=%u (now=%u)", + entry.get(), entry->GetExpirationTime(), now)); + continue; + } + + // not purged, move to the next one + ++i; + } +} + +void +CacheStorageService::MemoryPool::PurgeByFrecency(bool &aFrecencyNeedsSort, uint32_t aWhat) +{ + MOZ_ASSERT(IsOnManagementThread()); + + if (aFrecencyNeedsSort) { + mFrecencyArray.Sort(FrecencyComparator()); + aFrecencyNeedsSort = false; + } + + uint32_t const memoryLimit = Limit(); + + for (uint32_t i = 0; mMemorySize > memoryLimit && i < mFrecencyArray.Length();) { + if (CacheIOThread::YieldAndRerun()) + return; + + RefPtr entry = mFrecencyArray[i]; + + if (entry->Purge(aWhat)) { + LOG((" abandoned (%d), entry=%p, frecency=%1.10f", + aWhat, entry.get(), entry->GetFrecency())); + continue; + } + + // not purged, move to the next one + ++i; + } +} + +void +CacheStorageService::MemoryPool::PurgeAll(uint32_t aWhat) +{ + LOG(("CacheStorageService::MemoryPool::PurgeAll aWhat=%d", aWhat)); + MOZ_ASSERT(IsOnManagementThread()); + + for (uint32_t i = 0; i < mFrecencyArray.Length();) { + if (CacheIOThread::YieldAndRerun()) + return; + + RefPtr entry = mFrecencyArray[i]; + + if (entry->Purge(aWhat)) { + LOG((" abandoned entry=%p", entry.get())); + continue; + } + + // not purged, move to the next one + ++i; + } +} + +// Methods exposed to and used by CacheStorage. + +nsresult +CacheStorageService::AddStorageEntry(CacheStorage const* aStorage, + const nsACString & aURI, + const nsACString & aIdExtension, + bool aReplace, + CacheEntryHandle** aResult) +{ + NS_ENSURE_FALSE(mShutdown, NS_ERROR_NOT_INITIALIZED); + + NS_ENSURE_ARG(aStorage); + + nsAutoCString contextKey; + CacheFileUtils::AppendKeyPrefix(aStorage->LoadInfo(), contextKey); + + return AddStorageEntry(contextKey, aURI, aIdExtension, + aStorage->WriteToDisk(), + aStorage->SkipSizeCheck(), + aStorage->Pinning(), + aReplace, + aResult); +} + +nsresult +CacheStorageService::AddStorageEntry(nsCSubstring const& aContextKey, + const nsACString & aURI, + const nsACString & aIdExtension, + bool aWriteToDisk, + bool aSkipSizeCheck, + bool aPin, + bool aReplace, + CacheEntryHandle** aResult) +{ + nsresult rv; + + nsAutoCString entryKey; + rv = CacheEntry::HashingKey(EmptyCString(), aIdExtension, aURI, entryKey); + NS_ENSURE_SUCCESS(rv, rv); + + LOG(("CacheStorageService::AddStorageEntry [entryKey=%s, contextKey=%s]", + entryKey.get(), aContextKey.BeginReading())); + + RefPtr entry; + RefPtr handle; + + { + mozilla::MutexAutoLock lock(mLock); + + NS_ENSURE_FALSE(mShutdown, NS_ERROR_NOT_INITIALIZED); + + // Ensure storage table + CacheEntryTable* entries; + if (!sGlobalEntryTables->Get(aContextKey, &entries)) { + entries = new CacheEntryTable(CacheEntryTable::ALL_ENTRIES); + sGlobalEntryTables->Put(aContextKey, entries); + LOG((" new storage entries table for context '%s'", aContextKey.BeginReading())); + } + + bool entryExists = entries->Get(entryKey, getter_AddRefs(entry)); + + if (entryExists && !aReplace) { + // check whether we want to turn this entry to a memory-only. + if (MOZ_UNLIKELY(!aWriteToDisk) && MOZ_LIKELY(entry->IsUsingDisk())) { + LOG((" entry is persistent but we want mem-only, replacing it")); + aReplace = true; + } + } + + // If truncate is demanded, delete and doom the current entry + if (entryExists && aReplace) { + entries->Remove(entryKey); + + LOG((" dooming entry %p for %s because of OPEN_TRUNCATE", entry.get(), entryKey.get())); + // On purpose called under the lock to prevent races of doom and open on I/O thread + // No need to remove from both memory-only and all-entries tables. The new entry + // will overwrite the shadow entry in its ctor. + entry->DoomAlreadyRemoved(); + + entry = nullptr; + entryExists = false; + + // Would only lead to deleting force-valid timestamp again. We don't need the + // replace information anymore after this point anyway. + aReplace = false; + } + + // Ensure entry for the particular URL + if (!entryExists) { + // When replacing with a new entry, always remove the current force-valid timestamp, + // this is the only place to do it. + if (aReplace) { + RemoveEntryForceValid(aContextKey, entryKey); + } + + // Entry is not in the hashtable or has just been truncated... + entry = new CacheEntry(aContextKey, aURI, aIdExtension, aWriteToDisk, aSkipSizeCheck, aPin); + entries->Put(entryKey, entry); + LOG((" new entry %p for %s", entry.get(), entryKey.get())); + } + + if (entry) { + // Here, if this entry was not for a long time referenced by any consumer, + // gets again first 'handles count' reference. + handle = entry->NewHandle(); + } + } + + handle.forget(aResult); + return NS_OK; +} + +nsresult +CacheStorageService::CheckStorageEntry(CacheStorage const* aStorage, + const nsACString & aURI, + const nsACString & aIdExtension, + bool* aResult) +{ + nsresult rv; + + nsAutoCString contextKey; + CacheFileUtils::AppendKeyPrefix(aStorage->LoadInfo(), contextKey); + + if (!aStorage->WriteToDisk()) { + AppendMemoryStorageID(contextKey); + } + + LOG(("CacheStorageService::CheckStorageEntry [uri=%s, eid=%s, contextKey=%s]", + aURI.BeginReading(), aIdExtension.BeginReading(), contextKey.get())); + + { + mozilla::MutexAutoLock lock(mLock); + + NS_ENSURE_FALSE(mShutdown, NS_ERROR_NOT_INITIALIZED); + + nsAutoCString entryKey; + rv = CacheEntry::HashingKey(EmptyCString(), aIdExtension, aURI, entryKey); + NS_ENSURE_SUCCESS(rv, rv); + + CacheEntryTable* entries; + if ((*aResult = sGlobalEntryTables->Get(contextKey, &entries)) && + entries->GetWeak(entryKey, aResult)) { + LOG((" found in hash tables")); + return NS_OK; + } + } + + if (!aStorage->WriteToDisk()) { + // Memory entry, nothing more to do. + LOG((" not found in hash tables")); + return NS_OK; + } + + // Disk entry, not found in the hashtable, check the index. + nsAutoCString fileKey; + rv = CacheEntry::HashingKey(contextKey, aIdExtension, aURI, fileKey); + + CacheIndex::EntryStatus status; + rv = CacheIndex::HasEntry(fileKey, &status); + if (NS_FAILED(rv) || status == CacheIndex::DO_NOT_KNOW) { + LOG((" index doesn't know, rv=0x%08x", rv)); + return NS_ERROR_NOT_AVAILABLE; + } + + *aResult = status == CacheIndex::EXISTS; + LOG((" %sfound in index", *aResult ? "" : "not ")); + return NS_OK; +} + +namespace { + +class CacheEntryDoomByKeyCallback : public CacheFileIOListener + , public nsIRunnable +{ +public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSIRUNNABLE + + explicit CacheEntryDoomByKeyCallback(nsICacheEntryDoomCallback* aCallback) + : mCallback(aCallback) { } + +private: + virtual ~CacheEntryDoomByKeyCallback(); + + NS_IMETHOD OnFileOpened(CacheFileHandle *aHandle, nsresult aResult) override { return NS_OK; } + NS_IMETHOD OnDataWritten(CacheFileHandle *aHandle, const char *aBuf, nsresult aResult) override { return NS_OK; } + NS_IMETHOD OnDataRead(CacheFileHandle *aHandle, char *aBuf, nsresult aResult) override { return NS_OK; } + NS_IMETHOD OnFileDoomed(CacheFileHandle *aHandle, nsresult aResult) override; + NS_IMETHOD OnEOFSet(CacheFileHandle *aHandle, nsresult aResult) override { return NS_OK; } + NS_IMETHOD OnFileRenamed(CacheFileHandle *aHandle, nsresult aResult) override { return NS_OK; } + + nsCOMPtr mCallback; + nsresult mResult; +}; + +CacheEntryDoomByKeyCallback::~CacheEntryDoomByKeyCallback() +{ + if (mCallback) + ProxyReleaseMainThread(mCallback); +} + +NS_IMETHODIMP CacheEntryDoomByKeyCallback::OnFileDoomed(CacheFileHandle *aHandle, + nsresult aResult) +{ + if (!mCallback) + return NS_OK; + + mResult = aResult; + if (NS_IsMainThread()) { + Run(); + } else { + NS_DispatchToMainThread(this); + } + + return NS_OK; +} + +NS_IMETHODIMP CacheEntryDoomByKeyCallback::Run() +{ + mCallback->OnCacheEntryDoomed(mResult); + return NS_OK; +} + +NS_IMPL_ISUPPORTS(CacheEntryDoomByKeyCallback, CacheFileIOListener, nsIRunnable); + +} // namespace + +nsresult +CacheStorageService::DoomStorageEntry(CacheStorage const* aStorage, + const nsACString & aURI, + const nsACString & aIdExtension, + nsICacheEntryDoomCallback* aCallback) +{ + LOG(("CacheStorageService::DoomStorageEntry")); + + NS_ENSURE_ARG(aStorage); + + nsAutoCString contextKey; + CacheFileUtils::AppendKeyPrefix(aStorage->LoadInfo(), contextKey); + + nsAutoCString entryKey; + nsresult rv = CacheEntry::HashingKey(EmptyCString(), aIdExtension, aURI, entryKey); + NS_ENSURE_SUCCESS(rv, rv); + + RefPtr entry; + { + mozilla::MutexAutoLock lock(mLock); + + NS_ENSURE_FALSE(mShutdown, NS_ERROR_NOT_INITIALIZED); + + CacheEntryTable* entries; + if (sGlobalEntryTables->Get(contextKey, &entries)) { + if (entries->Get(entryKey, getter_AddRefs(entry))) { + if (aStorage->WriteToDisk() || !entry->IsUsingDisk()) { + // When evicting from disk storage, purge + // When evicting from memory storage and the entry is memory-only, purge + LOG((" purging entry %p for %s [storage use disk=%d, entry use disk=%d]", + entry.get(), entryKey.get(), aStorage->WriteToDisk(), entry->IsUsingDisk())); + entries->Remove(entryKey); + } + else { + // Otherwise, leave it + LOG((" leaving entry %p for %s [storage use disk=%d, entry use disk=%d]", + entry.get(), entryKey.get(), aStorage->WriteToDisk(), entry->IsUsingDisk())); + entry = nullptr; + } + } + } + + if (!entry) { + RemoveEntryForceValid(contextKey, entryKey); + } + } + + if (entry) { + LOG((" dooming entry %p for %s", entry.get(), entryKey.get())); + return entry->AsyncDoom(aCallback); + } + + LOG((" no entry loaded for %s", entryKey.get())); + + if (aStorage->WriteToDisk()) { + nsAutoCString contextKey; + CacheFileUtils::AppendKeyPrefix(aStorage->LoadInfo(), contextKey); + + rv = CacheEntry::HashingKey(contextKey, aIdExtension, aURI, entryKey); + NS_ENSURE_SUCCESS(rv, rv); + + LOG((" dooming file only for %s", entryKey.get())); + + RefPtr callback( + new CacheEntryDoomByKeyCallback(aCallback)); + rv = CacheFileIOManager::DoomFileByKey(entryKey, callback); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; + } + + class Callback : public Runnable + { + public: + explicit Callback(nsICacheEntryDoomCallback* aCallback) : mCallback(aCallback) { } + NS_IMETHOD Run() override + { + mCallback->OnCacheEntryDoomed(NS_ERROR_NOT_AVAILABLE); + return NS_OK; + } + nsCOMPtr mCallback; + }; + + if (aCallback) { + RefPtr callback = new Callback(aCallback); + return NS_DispatchToMainThread(callback); + } + + return NS_OK; +} + +nsresult +CacheStorageService::DoomStorageEntries(CacheStorage const* aStorage, + nsICacheEntryDoomCallback* aCallback) +{ + LOG(("CacheStorageService::DoomStorageEntries")); + + NS_ENSURE_FALSE(mShutdown, NS_ERROR_NOT_INITIALIZED); + NS_ENSURE_ARG(aStorage); + + nsAutoCString contextKey; + CacheFileUtils::AppendKeyPrefix(aStorage->LoadInfo(), contextKey); + + mozilla::MutexAutoLock lock(mLock); + + return DoomStorageEntries(contextKey, aStorage->LoadInfo(), + aStorage->WriteToDisk(), aStorage->Pinning(), + aCallback); +} + +nsresult +CacheStorageService::DoomStorageEntries(nsCSubstring const& aContextKey, + nsILoadContextInfo* aContext, + bool aDiskStorage, + bool aPinned, + nsICacheEntryDoomCallback* aCallback) +{ + LOG(("CacheStorageService::DoomStorageEntries [context=%s]", aContextKey.BeginReading())); + + mLock.AssertCurrentThreadOwns(); + + NS_ENSURE_TRUE(!mShutdown, NS_ERROR_NOT_INITIALIZED); + + nsAutoCString memoryStorageID(aContextKey); + AppendMemoryStorageID(memoryStorageID); + + if (aDiskStorage) { + LOG((" dooming disk+memory storage of %s", aContextKey.BeginReading())); + + // Walk one by one and remove entries according their pin status + CacheEntryTable *diskEntries, *memoryEntries; + if (sGlobalEntryTables->Get(aContextKey, &diskEntries)) { + sGlobalEntryTables->Get(memoryStorageID, &memoryEntries); + + for (auto iter = diskEntries->Iter(); !iter.Done(); iter.Next()) { + auto entry = iter.Data(); + if (entry->DeferOrBypassRemovalOnPinStatus(aPinned)) { + continue; + } + + if (memoryEntries) { + RemoveExactEntry(memoryEntries, iter.Key(), entry, false); + } + iter.Remove(); + } + } + + if (aContext && !aContext->IsPrivate()) { + LOG((" dooming disk entries")); + CacheFileIOManager::EvictByContext(aContext, aPinned); + } + } else { + LOG((" dooming memory-only storage of %s", aContextKey.BeginReading())); + + // Remove the memory entries table from the global tables. + // Since we store memory entries also in the disk entries table + // we need to remove the memory entries from the disk table one + // by one manually. + nsAutoPtr memoryEntries; + sGlobalEntryTables->RemoveAndForget(memoryStorageID, memoryEntries); + + CacheEntryTable* diskEntries; + if (memoryEntries && sGlobalEntryTables->Get(aContextKey, &diskEntries)) { + for (auto iter = memoryEntries->Iter(); !iter.Done(); iter.Next()) { + auto entry = iter.Data(); + RemoveExactEntry(diskEntries, iter.Key(), entry, false); + } + } + } + + { + mozilla::MutexAutoLock lock(mForcedValidEntriesLock); + + if (aContext) { + for (auto iter = mForcedValidEntries.Iter(); !iter.Done(); iter.Next()) { + bool matches; + DebugOnly rv = CacheFileUtils::KeyMatchesLoadContextInfo( + iter.Key(), aContext, &matches); + MOZ_ASSERT(NS_SUCCEEDED(rv)); + + if (matches) { + iter.Remove(); + } + } + } else { + mForcedValidEntries.Clear(); + } + } + + // An artificial callback. This is a candidate for removal tho. In the new + // cache any 'doom' or 'evict' function ensures that the entry or entries + // being doomed is/are not accessible after the function returns. So there is + // probably no need for a callback - has no meaning. But for compatibility + // with the old cache that is still in the tree we keep the API similar to be + // able to make tests as well as other consumers work for now. + class Callback : public Runnable + { + public: + explicit Callback(nsICacheEntryDoomCallback* aCallback) : mCallback(aCallback) { } + NS_IMETHOD Run() override + { + mCallback->OnCacheEntryDoomed(NS_OK); + return NS_OK; + } + nsCOMPtr mCallback; + }; + + if (aCallback) { + RefPtr callback = new Callback(aCallback); + return NS_DispatchToMainThread(callback); + } + + return NS_OK; +} + +nsresult +CacheStorageService::WalkStorageEntries(CacheStorage const* aStorage, + bool aVisitEntries, + nsICacheStorageVisitor* aVisitor) +{ + LOG(("CacheStorageService::WalkStorageEntries [cb=%p, visitentries=%d]", aVisitor, aVisitEntries)); + NS_ENSURE_FALSE(mShutdown, NS_ERROR_NOT_INITIALIZED); + + NS_ENSURE_ARG(aStorage); + + if (aStorage->WriteToDisk()) { + RefPtr event = + new WalkDiskCacheRunnable(aStorage->LoadInfo(), aVisitEntries, aVisitor); + return event->Walk(); + } + + RefPtr event = + new WalkMemoryCacheRunnable(aStorage->LoadInfo(), aVisitEntries, aVisitor); + return event->Walk(); +} + +void +CacheStorageService::CacheFileDoomed(nsILoadContextInfo* aLoadContextInfo, + const nsACString & aIdExtension, + const nsACString & aURISpec) +{ + nsAutoCString contextKey; + CacheFileUtils::AppendKeyPrefix(aLoadContextInfo, contextKey); + + nsAutoCString entryKey; + CacheEntry::HashingKey(EmptyCString(), aIdExtension, aURISpec, entryKey); + + mozilla::MutexAutoLock lock(mLock); + + if (mShutdown) { + return; + } + + CacheEntryTable* entries; + RefPtr entry; + + if (sGlobalEntryTables->Get(contextKey, &entries) && + entries->Get(entryKey, getter_AddRefs(entry))) { + if (entry->IsFileDoomed()) { + // Need to remove under the lock to avoid possible race leading + // to duplication of the entry per its key. + RemoveExactEntry(entries, entryKey, entry, false); + entry->DoomAlreadyRemoved(); + } + + // Entry found, but it's not the entry that has been found doomed + // by the lower eviction layer. Just leave everything unchanged. + return; + } + + RemoveEntryForceValid(contextKey, entryKey); +} + +bool +CacheStorageService::GetCacheEntryInfo(nsILoadContextInfo* aLoadContextInfo, + const nsACString & aIdExtension, + const nsACString & aURISpec, + EntryInfoCallback *aCallback) +{ + nsAutoCString contextKey; + CacheFileUtils::AppendKeyPrefix(aLoadContextInfo, contextKey); + + nsAutoCString entryKey; + CacheEntry::HashingKey(EmptyCString(), aIdExtension, aURISpec, entryKey); + + RefPtr entry; + { + mozilla::MutexAutoLock lock(mLock); + + if (mShutdown) { + return false; + } + + CacheEntryTable* entries; + if (!sGlobalEntryTables->Get(contextKey, &entries)) { + return false; + } + + if (!entries->Get(entryKey, getter_AddRefs(entry))) { + return false; + } + } + + GetCacheEntryInfo(entry, aCallback); + return true; +} + +// static +void +CacheStorageService::GetCacheEntryInfo(CacheEntry* aEntry, + EntryInfoCallback *aCallback) +{ + nsCString const uriSpec = aEntry->GetURI(); + nsCString const enhanceId = aEntry->GetEnhanceID(); + + uint32_t dataSize; + if (NS_FAILED(aEntry->GetStorageDataSize(&dataSize))) { + dataSize = 0; + } + int32_t fetchCount; + if (NS_FAILED(aEntry->GetFetchCount(&fetchCount))) { + fetchCount = 0; + } + uint32_t lastModified; + if (NS_FAILED(aEntry->GetLastModified(&lastModified))) { + lastModified = 0; + } + uint32_t expirationTime; + if (NS_FAILED(aEntry->GetExpirationTime(&expirationTime))) { + expirationTime = 0; + } + + aCallback->OnEntryInfo(uriSpec, enhanceId, dataSize, + fetchCount, lastModified, expirationTime, + aEntry->IsPinned()); +} + +// static +uint32_t CacheStorageService::CacheQueueSize(bool highPriority) +{ + RefPtr thread = CacheFileIOManager::IOThread(); + MOZ_ASSERT(thread); + return thread->QueueSize(highPriority); +} + +// Telementry collection + +namespace { + +bool TelemetryEntryKey(CacheEntry const* entry, nsAutoCString& key) +{ + nsAutoCString entryKey; + nsresult rv = entry->HashingKey(entryKey); + if (NS_FAILED(rv)) + return false; + + if (entry->GetStorageID().IsEmpty()) { + // Hopefully this will be const-copied, saves some memory + key = entryKey; + } else { + key.Assign(entry->GetStorageID()); + key.Append(':'); + key.Append(entryKey); + } + + return true; +} + +} // namespace + +void +CacheStorageService::TelemetryPrune(TimeStamp &now) +{ + static TimeDuration const oneMinute = TimeDuration::FromSeconds(60); + static TimeStamp dontPruneUntil = now + oneMinute; + if (now < dontPruneUntil) + return; + + static TimeDuration const fifteenMinutes = TimeDuration::FromSeconds(900); + for (auto iter = mPurgeTimeStamps.Iter(); !iter.Done(); iter.Next()) { + if (now - iter.Data() > fifteenMinutes) { + // We are not interested in resurrection of entries after 15 minutes + // of time. This is also the limit for the telemetry. + iter.Remove(); + } + } + dontPruneUntil = now + oneMinute; +} + +void +CacheStorageService::TelemetryRecordEntryCreation(CacheEntry const* entry) +{ + MOZ_ASSERT(CacheStorageService::IsOnManagementThread()); + + nsAutoCString key; + if (!TelemetryEntryKey(entry, key)) + return; + + TimeStamp now = TimeStamp::NowLoRes(); + TelemetryPrune(now); + + // When an entry is craeted (registered actually) we check if there is + // a timestamp marked when this very same cache entry has been removed + // (deregistered) because of over-memory-limit purging. If there is such + // a timestamp found accumulate telemetry on how long the entry was away. + TimeStamp timeStamp; + if (!mPurgeTimeStamps.Get(key, &timeStamp)) + return; + + mPurgeTimeStamps.Remove(key); + + Telemetry::AccumulateTimeDelta(Telemetry::HTTP_CACHE_ENTRY_RELOAD_TIME, + timeStamp, TimeStamp::NowLoRes()); +} + +void +CacheStorageService::TelemetryRecordEntryRemoval(CacheEntry const* entry) +{ + MOZ_ASSERT(CacheStorageService::IsOnManagementThread()); + + // Doomed entries must not be considered, we are only interested in purged + // entries. Note that the mIsDoomed flag is always set before deregistration + // happens. + if (entry->IsDoomed()) + return; + + nsAutoCString key; + if (!TelemetryEntryKey(entry, key)) + return; + + // When an entry is removed (deregistered actually) we put a timestamp for this + // entry to the hashtable so that when the entry is created (registered) again + // we know how long it was away. Also accumulate number of AsyncOpen calls on + // the entry, this tells us how efficiently the pool actually works. + + TimeStamp now = TimeStamp::NowLoRes(); + TelemetryPrune(now); + mPurgeTimeStamps.Put(key, now); + + Telemetry::Accumulate(Telemetry::HTTP_CACHE_ENTRY_REUSE_COUNT, entry->UseCount()); + Telemetry::AccumulateTimeDelta(Telemetry::HTTP_CACHE_ENTRY_ALIVE_TIME, + entry->LoadStart(), TimeStamp::NowLoRes()); +} + +// nsIMemoryReporter + +size_t +CacheStorageService::SizeOfExcludingThis(mozilla::MallocSizeOf mallocSizeOf) const +{ + CacheStorageService::Self()->Lock().AssertCurrentThreadOwns(); + + size_t n = 0; + // The elemets are referenced by sGlobalEntryTables and are reported from there + n += Pool(true).mFrecencyArray.ShallowSizeOfExcludingThis(mallocSizeOf); + n += Pool(true).mExpirationArray.ShallowSizeOfExcludingThis(mallocSizeOf); + n += Pool(false).mFrecencyArray.ShallowSizeOfExcludingThis(mallocSizeOf); + n += Pool(false).mExpirationArray.ShallowSizeOfExcludingThis(mallocSizeOf); + // Entries reported manually in CacheStorageService::CollectReports callback + if (sGlobalEntryTables) { + n += sGlobalEntryTables->ShallowSizeOfIncludingThis(mallocSizeOf); + } + + return n; +} + +size_t +CacheStorageService::SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const +{ + return mallocSizeOf(this) + SizeOfExcludingThis(mallocSizeOf); +} + +NS_IMETHODIMP +CacheStorageService::CollectReports(nsIHandleReportCallback* aHandleReport, + nsISupports* aData, bool aAnonymize) +{ + MOZ_COLLECT_REPORT( + "explicit/network/cache2/io", KIND_HEAP, UNITS_BYTES, + CacheFileIOManager::SizeOfIncludingThis(MallocSizeOf), + "Memory used by the cache IO manager."); + + MOZ_COLLECT_REPORT( + "explicit/network/cache2/index", KIND_HEAP, UNITS_BYTES, + CacheIndex::SizeOfIncludingThis(MallocSizeOf), + "Memory used by the cache index."); + + MutexAutoLock lock(mLock); + + // Report the service instance, this doesn't report entries, done lower + MOZ_COLLECT_REPORT( + "explicit/network/cache2/service", KIND_HEAP, UNITS_BYTES, + SizeOfIncludingThis(MallocSizeOf), + "Memory used by the cache storage service."); + + // Report all entries, each storage separately (by the context key) + // + // References are: + // sGlobalEntryTables to N CacheEntryTable + // CacheEntryTable to N CacheEntry + // CacheEntry to 1 CacheFile + // CacheFile to + // N CacheFileChunk (keeping the actual data) + // 1 CacheFileMetadata (keeping http headers etc.) + // 1 CacheFileOutputStream + // N CacheFileInputStream + if (sGlobalEntryTables) { + for (auto iter1 = sGlobalEntryTables->Iter(); !iter1.Done(); iter1.Next()) { + CacheStorageService::Self()->Lock().AssertCurrentThreadOwns(); + + CacheEntryTable* table = iter1.UserData(); + + size_t size = 0; + mozilla::MallocSizeOf mallocSizeOf = CacheStorageService::MallocSizeOf; + + size += table->ShallowSizeOfIncludingThis(mallocSizeOf); + for (auto iter2 = table->Iter(); !iter2.Done(); iter2.Next()) { + size += iter2.Key().SizeOfExcludingThisIfUnshared(mallocSizeOf); + + // Bypass memory-only entries, those will be reported when iterating the + // memory only table. Memory-only entries are stored in both ALL_ENTRIES + // and MEMORY_ONLY hashtables. + RefPtr const& entry = iter2.Data(); + if (table->Type() == CacheEntryTable::MEMORY_ONLY || + entry->IsUsingDisk()) { + size += entry->SizeOfIncludingThis(mallocSizeOf); + } + } + + // These key names are not privacy-sensitive. + aHandleReport->Callback( + EmptyCString(), + nsPrintfCString("explicit/network/cache2/%s-storage(%s)", + table->Type() == CacheEntryTable::MEMORY_ONLY ? "memory" : "disk", + iter1.Key().BeginReading()), + nsIMemoryReporter::KIND_HEAP, nsIMemoryReporter::UNITS_BYTES, size, + NS_LITERAL_CSTRING("Memory used by the cache storage."), + aData); + } + } + + return NS_OK; +} + +// nsICacheTesting + +NS_IMETHODIMP +CacheStorageService::IOThreadSuspender::Run() +{ + MonitorAutoLock mon(mMon); + while (!mSignaled) { + mon.Wait(); + } + return NS_OK; +} + +void +CacheStorageService::IOThreadSuspender::Notify() +{ + MonitorAutoLock mon(mMon); + mSignaled = true; + mon.Notify(); +} + +NS_IMETHODIMP +CacheStorageService::SuspendCacheIOThread(uint32_t aLevel) +{ + RefPtr thread = CacheFileIOManager::IOThread(); + if (!thread) { + return NS_ERROR_NOT_AVAILABLE; + } + + MOZ_ASSERT(!mActiveIOSuspender); + mActiveIOSuspender = new IOThreadSuspender(); + return thread->Dispatch(mActiveIOSuspender, aLevel); +} + +NS_IMETHODIMP +CacheStorageService::ResumeCacheIOThread() +{ + MOZ_ASSERT(mActiveIOSuspender); + + RefPtr suspender; + suspender.swap(mActiveIOSuspender); + suspender->Notify(); + return NS_OK; +} + +NS_IMETHODIMP +CacheStorageService::Flush(nsIObserver* aObserver) +{ + RefPtr thread = CacheFileIOManager::IOThread(); + if (!thread) { + return NS_ERROR_NOT_AVAILABLE; + } + + nsCOMPtr observerService = + mozilla::services::GetObserverService(); + if (!observerService) { + return NS_ERROR_NOT_AVAILABLE; + } + + // Adding as weak, the consumer is responsible to keep the reference + // until notified. + observerService->AddObserver(aObserver, "cacheservice:purge-memory-pools", false); + + // This runnable will do the purging and when done, notifies the above observer. + // We dispatch it to the CLOSE level, so all data writes scheduled up to this time + // will be done before this purging happens. + RefPtr r = + new CacheStorageService::PurgeFromMemoryRunnable(this, CacheEntry::PURGE_WHOLE); + + return thread->Dispatch(r, CacheIOThread::WRITE); +} + +} // namespace net +} // namespace mozilla -- cgit v1.2.3