diff options
author | Matt A. Tobin <mattatobin@localhost.localdomain> | 2018-02-02 04:16:08 -0500 |
---|---|---|
committer | Matt A. Tobin <mattatobin@localhost.localdomain> | 2018-02-02 04:16:08 -0500 |
commit | 5f8de423f190bbb79a62f804151bc24824fa32d8 (patch) | |
tree | 10027f336435511475e392454359edea8e25895d /netwerk/cache | |
parent | 49ee0794b5d912db1f95dce6eb52d781dc210db5 (diff) | |
download | UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.gz UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.lz UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.xz UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.zip |
Add m-esr52 at 52.6.0
Diffstat (limited to 'netwerk/cache')
45 files changed, 17835 insertions, 0 deletions
diff --git a/netwerk/cache/moz.build b/netwerk/cache/moz.build new file mode 100644 index 000000000..adf6e8bd2 --- /dev/null +++ b/netwerk/cache/moz.build @@ -0,0 +1,51 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +XPIDL_SOURCES += [ + 'nsICache.idl', + 'nsICacheEntryDescriptor.idl', + 'nsICacheListener.idl', + 'nsICacheService.idl', + 'nsICacheSession.idl', + 'nsICacheVisitor.idl', +] + +XPIDL_MODULE = 'necko_cache' + +EXPORTS += [ + 'nsApplicationCacheService.h', + 'nsCacheService.h', + 'nsDeleteDir.h' +] + +UNIFIED_SOURCES += [ + 'nsApplicationCacheService.cpp', + 'nsCache.cpp', + 'nsCacheEntry.cpp', + 'nsCacheEntryDescriptor.cpp', + 'nsCacheMetaData.cpp', + 'nsCacheService.cpp', + 'nsCacheSession.cpp', + 'nsCacheUtils.cpp', + 'nsDeleteDir.cpp', + 'nsDiskCacheBinding.cpp', + 'nsDiskCacheBlockFile.cpp', + 'nsDiskCacheDevice.cpp', + 'nsDiskCacheDeviceSQL.cpp', + 'nsDiskCacheEntry.cpp', + 'nsDiskCacheMap.cpp', + 'nsDiskCacheStreams.cpp', + 'nsMemoryCacheDevice.cpp', +] + +FINAL_LIBRARY = 'xul' + +LOCAL_INCLUDES += [ + '/netwerk/base', +] + +if CONFIG['GNU_CXX']: + CXXFLAGS += ['-Wno-error=shadow'] diff --git a/netwerk/cache/nsApplicationCache.h b/netwerk/cache/nsApplicationCache.h new file mode 100644 index 000000000..fc8501005 --- /dev/null +++ b/netwerk/cache/nsApplicationCache.h @@ -0,0 +1,29 @@ +/* vim:set ts=2 sw=2 sts=2 et cin: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +class nsApplicationCache : public nsIApplicationCache + , public nsSupportsWeakReference +{ +public: + NS_DECL_ISUPPORTS + NS_DECL_NSIAPPLICATIONCACHE + + nsApplicationCache(nsOfflineCacheDevice *device, + const nsACString &group, + const nsACString &clientID); + + nsApplicationCache(); + + void MarkInvalid(); + +private: + virtual ~nsApplicationCache(); + + RefPtr<nsOfflineCacheDevice> mDevice; + nsCString mGroup; + nsCString mClientID; + bool mValid; +}; + diff --git a/netwerk/cache/nsApplicationCacheService.cpp b/netwerk/cache/nsApplicationCacheService.cpp new file mode 100644 index 000000000..17012518d --- /dev/null +++ b/netwerk/cache/nsApplicationCacheService.cpp @@ -0,0 +1,268 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsDiskCache.h" +#include "nsDiskCacheDeviceSQL.h" +#include "nsCacheService.h" +#include "nsApplicationCacheService.h" +#include "nsCRT.h" +#include "mozIApplicationClearPrivateDataParams.h" +#include "nsNetCID.h" +#include "nsNetUtil.h" +#include "nsIObserverService.h" +#include "nsIPrincipal.h" +#include "mozilla/LoadContextInfo.h" + +using namespace mozilla; + +static NS_DEFINE_CID(kCacheServiceCID, NS_CACHESERVICE_CID); + +//----------------------------------------------------------------------------- +// nsApplicationCacheService +//----------------------------------------------------------------------------- + +NS_IMPL_ISUPPORTS(nsApplicationCacheService, nsIApplicationCacheService) + +nsApplicationCacheService::nsApplicationCacheService() +{ + nsCOMPtr<nsICacheService> serv = do_GetService(kCacheServiceCID); + mCacheService = nsCacheService::GlobalInstance(); +} + +nsApplicationCacheService::~nsApplicationCacheService() +{ +} + +NS_IMETHODIMP +nsApplicationCacheService::BuildGroupIDForInfo( + nsIURI *aManifestURL, + nsILoadContextInfo *aLoadContextInfo, + nsACString &_result) +{ + nsresult rv; + + nsAutoCString originSuffix; + if (aLoadContextInfo) { + aLoadContextInfo->OriginAttributesPtr()->CreateSuffix(originSuffix); + } + + rv = nsOfflineCacheDevice::BuildApplicationCacheGroupID( + aManifestURL, originSuffix, _result); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +NS_IMETHODIMP +nsApplicationCacheService::BuildGroupIDForSuffix( + nsIURI *aManifestURL, + nsACString const &aOriginSuffix, + nsACString &_result) +{ + nsresult rv; + + rv = nsOfflineCacheDevice::BuildApplicationCacheGroupID( + aManifestURL, aOriginSuffix, _result); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +NS_IMETHODIMP +nsApplicationCacheService::CreateApplicationCache(const nsACString &group, + nsIApplicationCache **out) +{ + if (!mCacheService) + return NS_ERROR_UNEXPECTED; + + RefPtr<nsOfflineCacheDevice> device; + nsresult rv = mCacheService->GetOfflineDevice(getter_AddRefs(device)); + NS_ENSURE_SUCCESS(rv, rv); + return device->CreateApplicationCache(group, out); +} + +NS_IMETHODIMP +nsApplicationCacheService::CreateCustomApplicationCache(const nsACString & group, + nsIFile *profileDir, + int32_t quota, + nsIApplicationCache **out) +{ + if (!mCacheService) + return NS_ERROR_UNEXPECTED; + + RefPtr<nsOfflineCacheDevice> device; + nsresult rv = mCacheService->GetCustomOfflineDevice(profileDir, + quota, + getter_AddRefs(device)); + NS_ENSURE_SUCCESS(rv, rv); + return device->CreateApplicationCache(group, out); +} + +NS_IMETHODIMP +nsApplicationCacheService::GetApplicationCache(const nsACString &clientID, + nsIApplicationCache **out) +{ + if (!mCacheService) + return NS_ERROR_UNEXPECTED; + + RefPtr<nsOfflineCacheDevice> device; + nsresult rv = mCacheService->GetOfflineDevice(getter_AddRefs(device)); + NS_ENSURE_SUCCESS(rv, rv); + return device->GetApplicationCache(clientID, out); +} + +NS_IMETHODIMP +nsApplicationCacheService::GetActiveCache(const nsACString &group, + nsIApplicationCache **out) +{ + if (!mCacheService) + return NS_ERROR_UNEXPECTED; + + RefPtr<nsOfflineCacheDevice> device; + nsresult rv = mCacheService->GetOfflineDevice(getter_AddRefs(device)); + NS_ENSURE_SUCCESS(rv, rv); + return device->GetActiveCache(group, out); +} + +NS_IMETHODIMP +nsApplicationCacheService::DeactivateGroup(const nsACString &group) +{ + if (!mCacheService) + return NS_ERROR_UNEXPECTED; + + RefPtr<nsOfflineCacheDevice> device; + nsresult rv = mCacheService->GetOfflineDevice(getter_AddRefs(device)); + NS_ENSURE_SUCCESS(rv, rv); + return device->DeactivateGroup(group); +} + +NS_IMETHODIMP +nsApplicationCacheService::ChooseApplicationCache(const nsACString &key, + nsILoadContextInfo *aLoadContextInfo, + nsIApplicationCache **out) +{ + if (!mCacheService) + return NS_ERROR_UNEXPECTED; + + RefPtr<nsOfflineCacheDevice> device; + nsresult rv = mCacheService->GetOfflineDevice(getter_AddRefs(device)); + NS_ENSURE_SUCCESS(rv, rv); + + return device->ChooseApplicationCache(key, aLoadContextInfo, out); +} + +NS_IMETHODIMP +nsApplicationCacheService::CacheOpportunistically(nsIApplicationCache* cache, + const nsACString &key) +{ + if (!mCacheService) + return NS_ERROR_UNEXPECTED; + + RefPtr<nsOfflineCacheDevice> device; + nsresult rv = mCacheService->GetOfflineDevice(getter_AddRefs(device)); + NS_ENSURE_SUCCESS(rv, rv); + return device->CacheOpportunistically(cache, key); +} + +NS_IMETHODIMP +nsApplicationCacheService::Evict(nsILoadContextInfo *aInfo) +{ + if (!mCacheService) + return NS_ERROR_UNEXPECTED; + + RefPtr<nsOfflineCacheDevice> device; + nsresult rv = mCacheService->GetOfflineDevice(getter_AddRefs(device)); + NS_ENSURE_SUCCESS(rv, rv); + return device->Evict(aInfo); +} + +NS_IMETHODIMP +nsApplicationCacheService::EvictMatchingOriginAttributes(nsAString const &aPattern) +{ + if (!mCacheService) + return NS_ERROR_UNEXPECTED; + + RefPtr<nsOfflineCacheDevice> device; + nsresult rv = mCacheService->GetOfflineDevice(getter_AddRefs(device)); + NS_ENSURE_SUCCESS(rv, rv); + + mozilla::OriginAttributesPattern pattern; + if (!pattern.Init(aPattern)) { + NS_ERROR("Could not parse OriginAttributesPattern JSON in clear-origin-attributes-data notification"); + return NS_ERROR_FAILURE; + } + + return device->Evict(pattern); +} + +NS_IMETHODIMP +nsApplicationCacheService::GetGroups(uint32_t *count, + char ***keys) +{ + if (!mCacheService) + return NS_ERROR_UNEXPECTED; + + RefPtr<nsOfflineCacheDevice> device; + nsresult rv = mCacheService->GetOfflineDevice(getter_AddRefs(device)); + NS_ENSURE_SUCCESS(rv, rv); + return device->GetGroups(count, keys); +} + +NS_IMETHODIMP +nsApplicationCacheService::GetGroupsTimeOrdered(uint32_t *count, + char ***keys) +{ + if (!mCacheService) + return NS_ERROR_UNEXPECTED; + + RefPtr<nsOfflineCacheDevice> device; + nsresult rv = mCacheService->GetOfflineDevice(getter_AddRefs(device)); + NS_ENSURE_SUCCESS(rv, rv); + return device->GetGroupsTimeOrdered(count, keys); +} + +//----------------------------------------------------------------------------- +// AppCacheClearDataObserver: handles clearing appcache data for app uninstall +// and clearing user data events. +//----------------------------------------------------------------------------- + +namespace { + +class AppCacheClearDataObserver final : public nsIObserver { +public: + NS_DECL_ISUPPORTS + + // nsIObserver implementation. + NS_IMETHOD + Observe(nsISupports *aSubject, const char *aTopic, const char16_t *aData) override + { + MOZ_ASSERT(!nsCRT::strcmp(aTopic, "clear-origin-attributes-data")); + + nsresult rv; + + nsCOMPtr<nsIApplicationCacheService> cacheService = + do_GetService(NS_APPLICATIONCACHESERVICE_CONTRACTID, &rv); + NS_ENSURE_SUCCESS(rv, rv); + + return cacheService->EvictMatchingOriginAttributes(nsDependentString(aData)); + } + +private: + ~AppCacheClearDataObserver() {} +}; + +NS_IMPL_ISUPPORTS(AppCacheClearDataObserver, nsIObserver) + +} // namespace + +// Instantiates and registers AppCacheClearDataObserver for notifications +void +nsApplicationCacheService::AppClearDataObserverInit() +{ + nsCOMPtr<nsIObserverService> observerService = services::GetObserverService(); + if (observerService) { + RefPtr<AppCacheClearDataObserver> obs = new AppCacheClearDataObserver(); + observerService->AddObserver(obs, "clear-origin-attributes-data", /*ownsWeak=*/ false); + } +} diff --git a/netwerk/cache/nsApplicationCacheService.h b/netwerk/cache/nsApplicationCacheService.h new file mode 100644 index 000000000..73bd32206 --- /dev/null +++ b/netwerk/cache/nsApplicationCacheService.h @@ -0,0 +1,28 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef _nsApplicationCacheService_h_ +#define _nsApplicationCacheService_h_ + +#include "nsIApplicationCacheService.h" +#include "mozilla/Attributes.h" + +class nsCacheService; + +class nsApplicationCacheService final : public nsIApplicationCacheService +{ +public: + nsApplicationCacheService(); + + NS_DECL_ISUPPORTS + NS_DECL_NSIAPPLICATIONCACHESERVICE + + static void AppClearDataObserverInit(); + +private: + ~nsApplicationCacheService(); + RefPtr<nsCacheService> mCacheService; +}; + +#endif // _nsApplicationCacheService_h_ diff --git a/netwerk/cache/nsCache.cpp b/netwerk/cache/nsCache.cpp new file mode 100644 index 000000000..7f5d6a071 --- /dev/null +++ b/netwerk/cache/nsCache.cpp @@ -0,0 +1,95 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsCache.h" +#include "nsReadableUtils.h" +#include "nsDependentSubstring.h" +#include "nsString.h" + + +/** + * Cache Service Utility Functions + */ + +mozilla::LazyLogModule gCacheLog("cache"); + +void +CacheLogPrintPath(mozilla::LogLevel level, const char * format, nsIFile * item) +{ + nsAutoCString path; + nsresult rv = item->GetNativePath(path); + if (NS_SUCCEEDED(rv)) { + MOZ_LOG(gCacheLog, level, (format, path.get())); + } else { + MOZ_LOG(gCacheLog, level, ("GetNativePath failed: %x", rv)); + } +} + + +uint32_t +SecondsFromPRTime(PRTime prTime) +{ + int64_t microSecondsPerSecond = PR_USEC_PER_SEC; + return uint32_t(prTime / microSecondsPerSecond); +} + + +PRTime +PRTimeFromSeconds(uint32_t seconds) +{ + int64_t intermediateResult = seconds; + PRTime prTime = intermediateResult * PR_USEC_PER_SEC; + return prTime; +} + + +nsresult +ClientIDFromCacheKey(const nsACString& key, char ** result) +{ + nsresult rv = NS_OK; + *result = nullptr; + + nsReadingIterator<char> colon; + key.BeginReading(colon); + + nsReadingIterator<char> start; + key.BeginReading(start); + + nsReadingIterator<char> end; + key.EndReading(end); + + if (FindCharInReadable(':', colon, end)) { + *result = ToNewCString( Substring(start, colon)); + if (!*result) rv = NS_ERROR_OUT_OF_MEMORY; + } else { + NS_ASSERTION(false, "FindCharInRead failed to find ':'"); + rv = NS_ERROR_UNEXPECTED; + } + return rv; +} + + +nsresult +ClientKeyFromCacheKey(const nsCString& key, nsACString &result) +{ + nsresult rv = NS_OK; + + nsReadingIterator<char> start; + key.BeginReading(start); + + nsReadingIterator<char> end; + key.EndReading(end); + + if (FindCharInReadable(':', start, end)) { + ++start; // advance past clientID ':' delimiter + result.Assign(Substring(start, end)); + } else { + NS_ASSERTION(false, "FindCharInRead failed to find ':'"); + rv = NS_ERROR_UNEXPECTED; + result.Truncate(0); + } + return rv; +} diff --git a/netwerk/cache/nsCache.h b/netwerk/cache/nsCache.h new file mode 100644 index 000000000..3c4513140 --- /dev/null +++ b/netwerk/cache/nsCache.h @@ -0,0 +1,42 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +/** + * Cache Service Utility Functions + */ + +#ifndef _nsCache_h_ +#define _nsCache_h_ + +#include "mozilla/Logging.h" +#include "nsISupports.h" +#include "nsIFile.h" +#include "nsAString.h" +#include "prtime.h" +#include "nsError.h" + +// PR_LOG args = "format string", arg, arg, ... +extern mozilla::LazyLogModule gCacheLog; +void CacheLogPrintPath(mozilla::LogLevel level, + const char * format, + nsIFile * item); +#define CACHE_LOG_INFO(args) MOZ_LOG(gCacheLog, mozilla::LogLevel::Info, args) +#define CACHE_LOG_ERROR(args) MOZ_LOG(gCacheLog, mozilla::LogLevel::Error, args) +#define CACHE_LOG_WARNING(args) MOZ_LOG(gCacheLog, mozilla::LogLevel::Warning, args) +#define CACHE_LOG_DEBUG(args) MOZ_LOG(gCacheLog, mozilla::LogLevel::Debug, args) +#define CACHE_LOG_PATH(level, format, item) \ + CacheLogPrintPath(level, format, item) + + +extern uint32_t SecondsFromPRTime(PRTime prTime); +extern PRTime PRTimeFromSeconds(uint32_t seconds); + + +extern nsresult ClientIDFromCacheKey(const nsACString& key, char ** result); +extern nsresult ClientKeyFromCacheKey(const nsCString& key, nsACString &result); + + +#endif // _nsCache_h diff --git a/netwerk/cache/nsCacheDevice.h b/netwerk/cache/nsCacheDevice.h new file mode 100644 index 000000000..967c5a034 --- /dev/null +++ b/netwerk/cache/nsCacheDevice.h @@ -0,0 +1,63 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef _nsCacheDevice_h_ +#define _nsCacheDevice_h_ + +#include "nspr.h" +#include "nsError.h" +#include "nsICache.h" + +class nsIFile; +class nsCString; +class nsCacheEntry; +class nsICacheVisitor; +class nsIInputStream; +class nsIOutputStream; + +/****************************************************************************** +* nsCacheDevice +*******************************************************************************/ +class nsCacheDevice { +public: + nsCacheDevice() { MOZ_COUNT_CTOR(nsCacheDevice); } + virtual ~nsCacheDevice() { MOZ_COUNT_DTOR(nsCacheDevice); } + + virtual nsresult Init() = 0; + virtual nsresult Shutdown() = 0; + + virtual const char * GetDeviceID(void) = 0; + virtual nsCacheEntry * FindEntry( nsCString * key, bool *collision ) = 0; + + virtual nsresult DeactivateEntry( nsCacheEntry * entry ) = 0; + virtual nsresult BindEntry( nsCacheEntry * entry ) = 0; + virtual void DoomEntry( nsCacheEntry * entry ) = 0; + + virtual nsresult OpenInputStreamForEntry(nsCacheEntry * entry, + nsCacheAccessMode mode, + uint32_t offset, + nsIInputStream ** result) = 0; + + virtual nsresult OpenOutputStreamForEntry(nsCacheEntry * entry, + nsCacheAccessMode mode, + uint32_t offset, + nsIOutputStream ** result) = 0; + + virtual nsresult GetFileForEntry( nsCacheEntry * entry, + nsIFile ** result ) = 0; + + virtual nsresult OnDataSizeChange( nsCacheEntry * entry, int32_t deltaSize ) = 0; + + virtual nsresult Visit(nsICacheVisitor * visitor) = 0; + + /** + * Device must evict entries associated with clientID. If clientID == nullptr, all + * entries must be evicted. Active entries must be doomed, rather than evicted. + */ + virtual nsresult EvictEntries(const char * clientID) = 0; +}; + +#endif // _nsCacheDevice_h_ diff --git a/netwerk/cache/nsCacheEntry.cpp b/netwerk/cache/nsCacheEntry.cpp new file mode 100644 index 000000000..6703394c7 --- /dev/null +++ b/netwerk/cache/nsCacheEntry.cpp @@ -0,0 +1,508 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + + +#include "nsCache.h" +#include "nspr.h" +#include "nsCacheEntry.h" +#include "nsCacheEntryDescriptor.h" +#include "nsCacheMetaData.h" +#include "nsCacheRequest.h" +#include "nsThreadUtils.h" +#include "nsError.h" +#include "nsICacheService.h" +#include "nsCacheService.h" +#include "nsCacheDevice.h" +#include "nsHashKeys.h" + +using namespace mozilla; + +nsCacheEntry::nsCacheEntry(const nsACString & key, + bool streamBased, + nsCacheStoragePolicy storagePolicy) + : mKey(key), + mFetchCount(0), + mLastFetched(0), + mLastModified(0), + mExpirationTime(nsICache::NO_EXPIRATION_TIME), + mFlags(0), + mPredictedDataSize(-1), + mDataSize(0), + mCacheDevice(nullptr), + mCustomDevice(nullptr), + mData(nullptr) +{ + MOZ_COUNT_CTOR(nsCacheEntry); + PR_INIT_CLIST(this); + PR_INIT_CLIST(&mRequestQ); + PR_INIT_CLIST(&mDescriptorQ); + + if (streamBased) MarkStreamBased(); + SetStoragePolicy(storagePolicy); + + MarkPublic(); +} + + +nsCacheEntry::~nsCacheEntry() +{ + MOZ_COUNT_DTOR(nsCacheEntry); + + if (mData) + nsCacheService::ReleaseObject_Locked(mData, mThread); +} + + +nsresult +nsCacheEntry::Create( const char * key, + bool streamBased, + nsCacheStoragePolicy storagePolicy, + nsCacheDevice * device, + nsCacheEntry ** result) +{ + nsCacheEntry* entry = new nsCacheEntry(nsCString(key), + streamBased, + storagePolicy); + entry->SetCacheDevice(device); + *result = entry; + return NS_OK; +} + + +void +nsCacheEntry::Fetched() +{ + mLastFetched = SecondsFromPRTime(PR_Now()); + ++mFetchCount; + MarkEntryDirty(); +} + + +const char * +nsCacheEntry::GetDeviceID() +{ + if (mCacheDevice) return mCacheDevice->GetDeviceID(); + return nullptr; +} + + +void +nsCacheEntry::TouchData() +{ + mLastModified = SecondsFromPRTime(PR_Now()); + MarkDataDirty(); +} + + +void +nsCacheEntry::SetData(nsISupports * data) +{ + if (mData) { + nsCacheService::ReleaseObject_Locked(mData, mThread); + mData = nullptr; + } + + if (data) { + NS_ADDREF(mData = data); + mThread = do_GetCurrentThread(); + } +} + + +void +nsCacheEntry::TouchMetaData() +{ + mLastModified = SecondsFromPRTime(PR_Now()); + MarkMetaDataDirty(); +} + + +/** + * cache entry states + * 0 descriptors (new entry) + * 0 descriptors (existing, bound entry) + * n descriptors (existing, bound entry) valid + * n descriptors (existing, bound entry) not valid (wait until valid or doomed) + */ + +nsresult +nsCacheEntry::RequestAccess(nsCacheRequest * request, nsCacheAccessMode *accessGranted) +{ + nsresult rv = NS_OK; + + if (IsDoomed()) return NS_ERROR_CACHE_ENTRY_DOOMED; + + if (!IsInitialized()) { + // brand new, unbound entry + if (request->IsStreamBased()) MarkStreamBased(); + MarkInitialized(); + + *accessGranted = request->AccessRequested() & nsICache::ACCESS_WRITE; + NS_ASSERTION(*accessGranted, "new cache entry for READ-ONLY request"); + PR_APPEND_LINK(request, &mRequestQ); + return rv; + } + + if (IsStreamData() != request->IsStreamBased()) { + *accessGranted = nsICache::ACCESS_NONE; + return request->IsStreamBased() ? + NS_ERROR_CACHE_DATA_IS_NOT_STREAM : NS_ERROR_CACHE_DATA_IS_STREAM; + } + + if (PR_CLIST_IS_EMPTY(&mDescriptorQ)) { + // 1st descriptor for existing bound entry + *accessGranted = request->AccessRequested(); + if (*accessGranted & nsICache::ACCESS_WRITE) { + MarkInvalid(); + } else { + MarkValid(); + } + } else { + // nth request for existing, bound entry + *accessGranted = request->AccessRequested() & ~nsICache::ACCESS_WRITE; + if (!IsValid()) + rv = NS_ERROR_CACHE_WAIT_FOR_VALIDATION; + } + PR_APPEND_LINK(request,&mRequestQ); + + return rv; +} + + +nsresult +nsCacheEntry::CreateDescriptor(nsCacheRequest * request, + nsCacheAccessMode accessGranted, + nsICacheEntryDescriptor ** result) +{ + NS_ENSURE_ARG_POINTER(request && result); + + nsCacheEntryDescriptor * descriptor = + new nsCacheEntryDescriptor(this, accessGranted); + + // XXX check request is on q + PR_REMOVE_AND_INIT_LINK(request); // remove request regardless of success + + if (descriptor == nullptr) + return NS_ERROR_OUT_OF_MEMORY; + + PR_APPEND_LINK(descriptor, &mDescriptorQ); + + CACHE_LOG_DEBUG((" descriptor %p created for request %p on entry %p\n", + descriptor, request, this)); + + NS_ADDREF(*result = descriptor); + return NS_OK; +} + + +bool +nsCacheEntry::RemoveRequest(nsCacheRequest * request) +{ + // XXX if debug: verify this request belongs to this entry + PR_REMOVE_AND_INIT_LINK(request); + + // return true if this entry should stay active + return !((PR_CLIST_IS_EMPTY(&mRequestQ)) && + (PR_CLIST_IS_EMPTY(&mDescriptorQ))); +} + + +bool +nsCacheEntry::RemoveDescriptor(nsCacheEntryDescriptor * descriptor, + bool * doomEntry) +{ + NS_ASSERTION(descriptor->CacheEntry() == this, "### Wrong cache entry!!"); + + *doomEntry = descriptor->ClearCacheEntry(); + + PR_REMOVE_AND_INIT_LINK(descriptor); + + if (!PR_CLIST_IS_EMPTY(&mDescriptorQ)) + return true; // stay active if we still have open descriptors + + if (PR_CLIST_IS_EMPTY(&mRequestQ)) + return false; // no descriptors or requests, we can deactivate + + return true; // find next best request to give a descriptor to +} + + +void +nsCacheEntry::DetachDescriptors() +{ + nsCacheEntryDescriptor * descriptor = + (nsCacheEntryDescriptor *)PR_LIST_HEAD(&mDescriptorQ); + + while (descriptor != &mDescriptorQ) { + nsCacheEntryDescriptor * nextDescriptor = + (nsCacheEntryDescriptor *)PR_NEXT_LINK(descriptor); + + descriptor->ClearCacheEntry(); + PR_REMOVE_AND_INIT_LINK(descriptor); + descriptor = nextDescriptor; + } +} + + +void +nsCacheEntry::GetDescriptors( + nsTArray<RefPtr<nsCacheEntryDescriptor> > &outDescriptors) +{ + nsCacheEntryDescriptor * descriptor = + (nsCacheEntryDescriptor *)PR_LIST_HEAD(&mDescriptorQ); + + while (descriptor != &mDescriptorQ) { + nsCacheEntryDescriptor * nextDescriptor = + (nsCacheEntryDescriptor *)PR_NEXT_LINK(descriptor); + + outDescriptors.AppendElement(descriptor); + descriptor = nextDescriptor; + } +} + + +/****************************************************************************** + * nsCacheEntryInfo - for implementing about:cache + *****************************************************************************/ + +NS_IMPL_ISUPPORTS(nsCacheEntryInfo, nsICacheEntryInfo) + + +NS_IMETHODIMP +nsCacheEntryInfo::GetClientID(char ** clientID) +{ + NS_ENSURE_ARG_POINTER(clientID); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + + return ClientIDFromCacheKey(*mCacheEntry->Key(), clientID); +} + + +NS_IMETHODIMP +nsCacheEntryInfo::GetDeviceID(char ** deviceID) +{ + NS_ENSURE_ARG_POINTER(deviceID); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + + *deviceID = NS_strdup(mCacheEntry->GetDeviceID()); + return *deviceID ? NS_OK : NS_ERROR_OUT_OF_MEMORY; +} + + +NS_IMETHODIMP +nsCacheEntryInfo::GetKey(nsACString &key) +{ + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + + return ClientKeyFromCacheKey(*mCacheEntry->Key(), key); +} + + +NS_IMETHODIMP +nsCacheEntryInfo::GetFetchCount(int32_t * fetchCount) +{ + NS_ENSURE_ARG_POINTER(fetchCount); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + + *fetchCount = mCacheEntry->FetchCount(); + return NS_OK; +} + + +NS_IMETHODIMP +nsCacheEntryInfo::GetLastFetched(uint32_t * lastFetched) +{ + NS_ENSURE_ARG_POINTER(lastFetched); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + + *lastFetched = mCacheEntry->LastFetched(); + return NS_OK; +} + + +NS_IMETHODIMP +nsCacheEntryInfo::GetLastModified(uint32_t * lastModified) +{ + NS_ENSURE_ARG_POINTER(lastModified); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + + *lastModified = mCacheEntry->LastModified(); + return NS_OK; +} + + +NS_IMETHODIMP +nsCacheEntryInfo::GetExpirationTime(uint32_t * expirationTime) +{ + NS_ENSURE_ARG_POINTER(expirationTime); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + + *expirationTime = mCacheEntry->ExpirationTime(); + return NS_OK; +} + + +NS_IMETHODIMP +nsCacheEntryInfo::GetDataSize(uint32_t * dataSize) +{ + NS_ENSURE_ARG_POINTER(dataSize); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + + *dataSize = mCacheEntry->DataSize(); + return NS_OK; +} + + +NS_IMETHODIMP +nsCacheEntryInfo::IsStreamBased(bool * result) +{ + NS_ENSURE_ARG_POINTER(result); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + + *result = mCacheEntry->IsStreamData(); + return NS_OK; +} + + +/****************************************************************************** + * nsCacheEntryHashTable + *****************************************************************************/ + +const PLDHashTableOps +nsCacheEntryHashTable::ops = +{ + HashKey, + MatchEntry, + MoveEntry, + ClearEntry +}; + + +nsCacheEntryHashTable::nsCacheEntryHashTable() + : table(&ops, sizeof(nsCacheEntryHashTableEntry), kInitialTableLength) + , initialized(false) +{ + MOZ_COUNT_CTOR(nsCacheEntryHashTable); +} + + +nsCacheEntryHashTable::~nsCacheEntryHashTable() +{ + MOZ_COUNT_DTOR(nsCacheEntryHashTable); + if (initialized) + Shutdown(); +} + + +void +nsCacheEntryHashTable::Init() +{ + table.ClearAndPrepareForLength(kInitialTableLength); + initialized = true; +} + +void +nsCacheEntryHashTable::Shutdown() +{ + if (initialized) { + table.ClearAndPrepareForLength(kInitialTableLength); + initialized = false; + } +} + + +nsCacheEntry * +nsCacheEntryHashTable::GetEntry( const nsCString * key) +{ + NS_ASSERTION(initialized, "nsCacheEntryHashTable not initialized"); + if (!initialized) return nullptr; + + PLDHashEntryHdr *hashEntry = table.Search(key); + return hashEntry ? ((nsCacheEntryHashTableEntry *)hashEntry)->cacheEntry + : nullptr; +} + + +nsresult +nsCacheEntryHashTable::AddEntry( nsCacheEntry *cacheEntry) +{ + PLDHashEntryHdr *hashEntry; + + NS_ASSERTION(initialized, "nsCacheEntryHashTable not initialized"); + if (!initialized) return NS_ERROR_NOT_INITIALIZED; + if (!cacheEntry) return NS_ERROR_NULL_POINTER; + + hashEntry = table.Add(&(cacheEntry->mKey), fallible); + + if (!hashEntry) + return NS_ERROR_FAILURE; + NS_ASSERTION(((nsCacheEntryHashTableEntry *)hashEntry)->cacheEntry == 0, + "### nsCacheEntryHashTable::AddEntry - entry already used"); + ((nsCacheEntryHashTableEntry *)hashEntry)->cacheEntry = cacheEntry; + + return NS_OK; +} + + +void +nsCacheEntryHashTable::RemoveEntry( nsCacheEntry *cacheEntry) +{ + NS_ASSERTION(initialized, "nsCacheEntryHashTable not initialized"); + NS_ASSERTION(cacheEntry, "### cacheEntry == nullptr"); + + if (!initialized) return; // NS_ERROR_NOT_INITIALIZED + +#if DEBUG + // XXX debug code to make sure we have the entry we're trying to remove + nsCacheEntry *check = GetEntry(&(cacheEntry->mKey)); + NS_ASSERTION(check == cacheEntry, "### Attempting to remove unknown cache entry!!!"); +#endif + table.Remove(&(cacheEntry->mKey)); +} + +PLDHashTable::Iterator +nsCacheEntryHashTable::Iter() +{ + return PLDHashTable::Iterator(&table); +} + +/** + * hash table operation callback functions + */ + +PLDHashNumber +nsCacheEntryHashTable::HashKey(const void *key) +{ + return HashString(*static_cast<const nsCString *>(key)); +} + +bool +nsCacheEntryHashTable::MatchEntry(const PLDHashEntryHdr * hashEntry, + const void * key) +{ + NS_ASSERTION(key != nullptr, "### nsCacheEntryHashTable::MatchEntry : null key"); + nsCacheEntry *cacheEntry = ((nsCacheEntryHashTableEntry *)hashEntry)->cacheEntry; + + return cacheEntry->mKey.Equals(*(nsCString *)key); +} + + +void +nsCacheEntryHashTable::MoveEntry(PLDHashTable * /* table */, + const PLDHashEntryHdr *from, + PLDHashEntryHdr *to) +{ + ((nsCacheEntryHashTableEntry *)to)->cacheEntry = + ((nsCacheEntryHashTableEntry *)from)->cacheEntry; +} + + +void +nsCacheEntryHashTable::ClearEntry(PLDHashTable * /* table */, + PLDHashEntryHdr * hashEntry) +{ + ((nsCacheEntryHashTableEntry *)hashEntry)->cacheEntry = 0; +} diff --git a/netwerk/cache/nsCacheEntry.h b/netwerk/cache/nsCacheEntry.h new file mode 100644 index 000000000..bef7d9ace --- /dev/null +++ b/netwerk/cache/nsCacheEntry.h @@ -0,0 +1,302 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef _nsCacheEntry_h_ +#define _nsCacheEntry_h_ + +#include "nsICache.h" +#include "nsICacheEntryDescriptor.h" +#include "nsIThread.h" +#include "nsCacheMetaData.h" + +#include "nspr.h" +#include "PLDHashTable.h" +#include "nsAutoPtr.h" +#include "nscore.h" +#include "nsCOMPtr.h" +#include "nsString.h" +#include "nsAString.h" + +class nsCacheDevice; +class nsCacheMetaData; +class nsCacheRequest; +class nsCacheEntryDescriptor; + +/****************************************************************************** +* nsCacheEntry +*******************************************************************************/ +class nsCacheEntry : public PRCList +{ +public: + + nsCacheEntry(const nsACString & key, + bool streamBased, + nsCacheStoragePolicy storagePolicy); + ~nsCacheEntry(); + + + static nsresult Create( const char * key, + bool streamBased, + nsCacheStoragePolicy storagePolicy, + nsCacheDevice * device, + nsCacheEntry ** result); + + nsCString * Key() { return &mKey; } + + int32_t FetchCount() { return mFetchCount; } + void SetFetchCount( int32_t count) { mFetchCount = count; } + void Fetched(); + + uint32_t LastFetched() { return mLastFetched; } + void SetLastFetched( uint32_t lastFetched) { mLastFetched = lastFetched; } + + uint32_t LastModified() { return mLastModified; } + void SetLastModified( uint32_t lastModified) { mLastModified = lastModified; } + + uint32_t ExpirationTime() { return mExpirationTime; } + void SetExpirationTime( uint32_t expires) { mExpirationTime = expires; } + + uint32_t Size() + { return mDataSize + mMetaData.Size() + mKey.Length() ; } + + nsCacheDevice * CacheDevice() { return mCacheDevice; } + void SetCacheDevice( nsCacheDevice * device) { mCacheDevice = device; } + void SetCustomCacheDevice( nsCacheDevice * device ) + { mCustomDevice = device; } + nsCacheDevice * CustomCacheDevice() { return mCustomDevice; } + const char * GetDeviceID(); + + /** + * Data accessors + */ + nsISupports *Data() { return mData; } + void SetData( nsISupports * data); + + int64_t PredictedDataSize() { return mPredictedDataSize; } + void SetPredictedDataSize(int64_t size) { mPredictedDataSize = size; } + + uint32_t DataSize() { return mDataSize; } + void SetDataSize( uint32_t size) { mDataSize = size; } + + void TouchData(); + + /** + * Meta data accessors + */ + const char * GetMetaDataElement( const char * key) { return mMetaData.GetElement(key); } + nsresult SetMetaDataElement( const char * key, + const char * value) { return mMetaData.SetElement(key, value); } + nsresult VisitMetaDataElements( nsICacheMetaDataVisitor * visitor) { return mMetaData.VisitElements(visitor); } + nsresult FlattenMetaData(char * buffer, uint32_t bufSize) { return mMetaData.FlattenMetaData(buffer, bufSize); } + nsresult UnflattenMetaData(const char * buffer, uint32_t bufSize) { return mMetaData.UnflattenMetaData(buffer, bufSize); } + uint32_t MetaDataSize() { return mMetaData.Size(); } + + void TouchMetaData(); + + + /** + * Security Info accessors + */ + nsISupports* SecurityInfo() { return mSecurityInfo; } + void SetSecurityInfo( nsISupports * info) { mSecurityInfo = info; } + + + // XXX enumerate MetaData method + + + enum CacheEntryFlags { + eStoragePolicyMask = 0x000000FF, + eDoomedMask = 0x00000100, + eEntryDirtyMask = 0x00000200, + eDataDirtyMask = 0x00000400, + eMetaDataDirtyMask = 0x00000800, + eStreamDataMask = 0x00001000, + eActiveMask = 0x00002000, + eInitializedMask = 0x00004000, + eValidMask = 0x00008000, + eBindingMask = 0x00010000, + ePrivateMask = 0x00020000 + }; + + void MarkBinding() { mFlags |= eBindingMask; } + void ClearBinding() { mFlags &= ~eBindingMask; } + bool IsBinding() { return (mFlags & eBindingMask) != 0; } + + void MarkEntryDirty() { mFlags |= eEntryDirtyMask; } + void MarkEntryClean() { mFlags &= ~eEntryDirtyMask; } + void MarkDataDirty() { mFlags |= eDataDirtyMask; } + void MarkDataClean() { mFlags &= ~eDataDirtyMask; } + void MarkMetaDataDirty() { mFlags |= eMetaDataDirtyMask; } + void MarkMetaDataClean() { mFlags &= ~eMetaDataDirtyMask; } + void MarkStreamData() { mFlags |= eStreamDataMask; } + void MarkValid() { mFlags |= eValidMask; } + void MarkInvalid() { mFlags &= ~eValidMask; } + void MarkPrivate() { mFlags |= ePrivateMask; } + void MarkPublic() { mFlags &= ~ePrivateMask; } + // void MarkAllowedInMemory() { mFlags |= eAllowedInMemoryMask; } + // void MarkAllowedOnDisk() { mFlags |= eAllowedOnDiskMask; } + + bool IsDoomed() { return (mFlags & eDoomedMask) != 0; } + bool IsEntryDirty() { return (mFlags & eEntryDirtyMask) != 0; } + bool IsDataDirty() { return (mFlags & eDataDirtyMask) != 0; } + bool IsMetaDataDirty() { return (mFlags & eMetaDataDirtyMask) != 0; } + bool IsStreamData() { return (mFlags & eStreamDataMask) != 0; } + bool IsActive() { return (mFlags & eActiveMask) != 0; } + bool IsInitialized() { return (mFlags & eInitializedMask) != 0; } + bool IsValid() { return (mFlags & eValidMask) != 0; } + bool IsInvalid() { return (mFlags & eValidMask) == 0; } + bool IsInUse() { return IsBinding() || + !(PR_CLIST_IS_EMPTY(&mRequestQ) && + PR_CLIST_IS_EMPTY(&mDescriptorQ)); } + bool IsNotInUse() { return !IsInUse(); } + bool IsPrivate() { return (mFlags & ePrivateMask) != 0; } + + + bool IsAllowedInMemory() + { + return (StoragePolicy() == nsICache::STORE_ANYWHERE) || + (StoragePolicy() == nsICache::STORE_IN_MEMORY); + } + + bool IsAllowedOnDisk() + { + return !IsPrivate() && ((StoragePolicy() == nsICache::STORE_ANYWHERE) || + (StoragePolicy() == nsICache::STORE_ON_DISK)); + } + + bool IsAllowedOffline() + { + return (StoragePolicy() == nsICache::STORE_OFFLINE); + } + + nsCacheStoragePolicy StoragePolicy() + { + return (nsCacheStoragePolicy)(mFlags & eStoragePolicyMask); + } + + void SetStoragePolicy(nsCacheStoragePolicy policy) + { + NS_ASSERTION(policy <= 0xFF, "too many bits in nsCacheStoragePolicy"); + mFlags &= ~eStoragePolicyMask; // clear storage policy bits + mFlags |= policy; + } + + + // methods for nsCacheService + nsresult RequestAccess( nsCacheRequest * request, nsCacheAccessMode *accessGranted); + nsresult CreateDescriptor( nsCacheRequest * request, + nsCacheAccessMode accessGranted, + nsICacheEntryDescriptor ** result); + + bool RemoveRequest( nsCacheRequest * request); + bool RemoveDescriptor( nsCacheEntryDescriptor * descriptor, + bool * doomEntry); + + void GetDescriptors(nsTArray<RefPtr<nsCacheEntryDescriptor> > &outDescriptors); + +private: + friend class nsCacheEntryHashTable; + friend class nsCacheService; + + void DetachDescriptors(); + + // internal methods + void MarkDoomed() { mFlags |= eDoomedMask; } + void MarkStreamBased() { mFlags |= eStreamDataMask; } + void MarkInitialized() { mFlags |= eInitializedMask; } + void MarkActive() { mFlags |= eActiveMask; } + void MarkInactive() { mFlags &= ~eActiveMask; } + + nsCString mKey; + uint32_t mFetchCount; // 4 + uint32_t mLastFetched; // 4 + uint32_t mLastModified; // 4 + uint32_t mLastValidated; // 4 + uint32_t mExpirationTime; // 4 + uint32_t mFlags; // 4 + int64_t mPredictedDataSize; // Size given by ContentLength. + uint32_t mDataSize; // 4 + nsCacheDevice * mCacheDevice; // 4 + nsCacheDevice * mCustomDevice; // 4 + nsCOMPtr<nsISupports> mSecurityInfo; // + nsISupports * mData; // strong ref + nsCOMPtr<nsIThread> mThread; + nsCacheMetaData mMetaData; // 4 + PRCList mRequestQ; // 8 + PRCList mDescriptorQ; // 8 +}; + + +/****************************************************************************** +* nsCacheEntryInfo +*******************************************************************************/ +class nsCacheEntryInfo : public nsICacheEntryInfo { +public: + NS_DECL_ISUPPORTS + NS_DECL_NSICACHEENTRYINFO + + explicit nsCacheEntryInfo(nsCacheEntry* entry) + : mCacheEntry(entry) + { + } + + void DetachEntry() { mCacheEntry = nullptr; } + +private: + nsCacheEntry * mCacheEntry; + + virtual ~nsCacheEntryInfo() {} +}; + + +/****************************************************************************** +* nsCacheEntryHashTable +*******************************************************************************/ + +struct nsCacheEntryHashTableEntry : public PLDHashEntryHdr +{ + nsCacheEntry *cacheEntry; +}; + +class nsCacheEntryHashTable +{ +public: + nsCacheEntryHashTable(); + ~nsCacheEntryHashTable(); + + void Init(); + void Shutdown(); + + nsCacheEntry *GetEntry( const nsCString * key); + nsresult AddEntry( nsCacheEntry *entry); + void RemoveEntry( nsCacheEntry *entry); + + PLDHashTable::Iterator Iter(); + +private: + // PLDHashTable operation callbacks + static PLDHashNumber HashKey(const void *key); + + static bool MatchEntry(const PLDHashEntryHdr * entry, + const void * key); + + static void MoveEntry( PLDHashTable *table, + const PLDHashEntryHdr *from, + PLDHashEntryHdr *to); + + static void ClearEntry( PLDHashTable *table, PLDHashEntryHdr *entry); + + static void Finalize( PLDHashTable *table); + + // member variables + static const PLDHashTableOps ops; + PLDHashTable table; + bool initialized; + + static const uint32_t kInitialTableLength = 256; +}; + +#endif // _nsCacheEntry_h_ diff --git a/netwerk/cache/nsCacheEntryDescriptor.cpp b/netwerk/cache/nsCacheEntryDescriptor.cpp new file mode 100644 index 000000000..b53ee3058 --- /dev/null +++ b/netwerk/cache/nsCacheEntryDescriptor.cpp @@ -0,0 +1,1475 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsICache.h" +#include "nsCache.h" +#include "nsCacheService.h" +#include "nsCacheEntryDescriptor.h" +#include "nsCacheEntry.h" +#include "nsReadableUtils.h" +#include "nsIOutputStream.h" +#include "nsCRT.h" +#include "nsThreadUtils.h" +#include <algorithm> + +#define kMinDecompressReadBufLen 1024 +#define kMinCompressWriteBufLen 1024 + + +/****************************************************************************** + * nsAsyncDoomEvent + *****************************************************************************/ + +class nsAsyncDoomEvent : public mozilla::Runnable { +public: + nsAsyncDoomEvent(nsCacheEntryDescriptor *descriptor, + nsICacheListener *listener) + { + mDescriptor = descriptor; + mListener = listener; + mThread = do_GetCurrentThread(); + // We addref the listener here and release it in nsNotifyDoomListener + // on the callers thread. If posting of nsNotifyDoomListener event fails + // we leak the listener which is better than releasing it on a wrong + // thread. + NS_IF_ADDREF(mListener); + } + + NS_IMETHOD Run() override + { + nsresult status = NS_OK; + + { + nsCacheServiceAutoLock lock(LOCK_TELEM(NSASYNCDOOMEVENT_RUN)); + + if (mDescriptor->mCacheEntry) { + status = nsCacheService::gService->DoomEntry_Internal( + mDescriptor->mCacheEntry, true); + } else if (!mDescriptor->mDoomedOnClose) { + status = NS_ERROR_NOT_AVAILABLE; + } + } + + if (mListener) { + mThread->Dispatch(new nsNotifyDoomListener(mListener, status), + NS_DISPATCH_NORMAL); + // posted event will release the reference on the correct thread + mListener = nullptr; + } + + return NS_OK; + } + +private: + RefPtr<nsCacheEntryDescriptor> mDescriptor; + nsICacheListener *mListener; + nsCOMPtr<nsIThread> mThread; +}; + + +NS_IMPL_ISUPPORTS(nsCacheEntryDescriptor, + nsICacheEntryDescriptor, + nsICacheEntryInfo) + +nsCacheEntryDescriptor::nsCacheEntryDescriptor(nsCacheEntry * entry, + nsCacheAccessMode accessGranted) + : mCacheEntry(entry), + mAccessGranted(accessGranted), + mOutputWrapper(nullptr), + mLock("nsCacheEntryDescriptor.mLock"), + mAsyncDoomPending(false), + mDoomedOnClose(false), + mClosingDescriptor(false) +{ + PR_INIT_CLIST(this); + NS_ADDREF(nsCacheService::GlobalInstance()); // ensure it lives for the lifetime of the descriptor +} + + +nsCacheEntryDescriptor::~nsCacheEntryDescriptor() +{ + // No need to close if the cache entry has already been severed. This + // helps avoid a shutdown assertion (bug 285519) that is caused when + // consumers end up holding onto these objects past xpcom-shutdown. It's + // okay for them to do that because the cache service calls our Close + // method during xpcom-shutdown, so we don't need to complain about it. + if (mCacheEntry) + Close(); + + NS_ASSERTION(mInputWrappers.IsEmpty(), + "We have still some input wrapper!"); + NS_ASSERTION(!mOutputWrapper, "We have still an output wrapper!"); + + nsCacheService * service = nsCacheService::GlobalInstance(); + NS_RELEASE(service); +} + + +NS_IMETHODIMP +nsCacheEntryDescriptor::GetClientID(char ** result) +{ + NS_ENSURE_ARG_POINTER(result); + + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHEENTRYDESCRIPTOR_GETCLIENTID)); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + + return ClientIDFromCacheKey(*(mCacheEntry->Key()), result); +} + + +NS_IMETHODIMP +nsCacheEntryDescriptor::GetDeviceID(char ** aDeviceID) +{ + NS_ENSURE_ARG_POINTER(aDeviceID); + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHEENTRYDESCRIPTOR_GETDEVICEID)); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + + const char* deviceID = mCacheEntry->GetDeviceID(); + if (!deviceID) { + *aDeviceID = nullptr; + return NS_OK; + } + + *aDeviceID = NS_strdup(deviceID); + return *aDeviceID ? NS_OK : NS_ERROR_OUT_OF_MEMORY; +} + + +NS_IMETHODIMP +nsCacheEntryDescriptor::GetKey(nsACString &result) +{ + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHEENTRYDESCRIPTOR_GETKEY)); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + + return ClientKeyFromCacheKey(*(mCacheEntry->Key()), result); +} + + +NS_IMETHODIMP +nsCacheEntryDescriptor::GetFetchCount(int32_t *result) +{ + NS_ENSURE_ARG_POINTER(result); + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHEENTRYDESCRIPTOR_GETFETCHCOUNT)); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + + *result = mCacheEntry->FetchCount(); + return NS_OK; +} + + +NS_IMETHODIMP +nsCacheEntryDescriptor::GetLastFetched(uint32_t *result) +{ + NS_ENSURE_ARG_POINTER(result); + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHEENTRYDESCRIPTOR_GETLASTFETCHED)); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + + *result = mCacheEntry->LastFetched(); + return NS_OK; +} + + +NS_IMETHODIMP +nsCacheEntryDescriptor::GetLastModified(uint32_t *result) +{ + NS_ENSURE_ARG_POINTER(result); + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHEENTRYDESCRIPTOR_GETLASTMODIFIED)); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + + *result = mCacheEntry->LastModified(); + return NS_OK; +} + + +NS_IMETHODIMP +nsCacheEntryDescriptor::GetExpirationTime(uint32_t *result) +{ + NS_ENSURE_ARG_POINTER(result); + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHEENTRYDESCRIPTOR_GETEXPIRATIONTIME)); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + + *result = mCacheEntry->ExpirationTime(); + return NS_OK; +} + + +NS_IMETHODIMP +nsCacheEntryDescriptor::SetExpirationTime(uint32_t expirationTime) +{ + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHEENTRYDESCRIPTOR_SETEXPIRATIONTIME)); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + + mCacheEntry->SetExpirationTime(expirationTime); + mCacheEntry->MarkEntryDirty(); + return NS_OK; +} + + +NS_IMETHODIMP nsCacheEntryDescriptor::IsStreamBased(bool *result) +{ + NS_ENSURE_ARG_POINTER(result); + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHEENTRYDESCRIPTOR_ISSTREAMBASED)); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + + *result = mCacheEntry->IsStreamData(); + return NS_OK; +} + +NS_IMETHODIMP nsCacheEntryDescriptor::GetPredictedDataSize(int64_t *result) +{ + NS_ENSURE_ARG_POINTER(result); + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHEENTRYDESCRIPTOR_GETPREDICTEDDATASIZE)); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + + *result = mCacheEntry->PredictedDataSize(); + return NS_OK; +} + +NS_IMETHODIMP nsCacheEntryDescriptor::SetPredictedDataSize(int64_t + predictedSize) +{ + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHEENTRYDESCRIPTOR_SETPREDICTEDDATASIZE)); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + + mCacheEntry->SetPredictedDataSize(predictedSize); + return NS_OK; +} + +NS_IMETHODIMP nsCacheEntryDescriptor::GetDataSize(uint32_t *result) +{ + NS_ENSURE_ARG_POINTER(result); + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHEENTRYDESCRIPTOR_GETDATASIZE)); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + + const char* val = mCacheEntry->GetMetaDataElement("uncompressed-len"); + if (!val) { + *result = mCacheEntry->DataSize(); + } else { + *result = atol(val); + } + + return NS_OK; +} + + +NS_IMETHODIMP nsCacheEntryDescriptor::GetStorageDataSize(uint32_t *result) +{ + NS_ENSURE_ARG_POINTER(result); + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHEENTRYDESCRIPTOR_GETSTORAGEDATASIZE)); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + + *result = mCacheEntry->DataSize(); + + return NS_OK; +} + + +nsresult +nsCacheEntryDescriptor::RequestDataSizeChange(int32_t deltaSize) +{ + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHEENTRYDESCRIPTOR_REQUESTDATASIZECHANGE)); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + + nsresult rv; + rv = nsCacheService::OnDataSizeChange(mCacheEntry, deltaSize); + if (NS_SUCCEEDED(rv)) { + // XXX review for signed/unsigned math errors + uint32_t newDataSize = mCacheEntry->DataSize() + deltaSize; + mCacheEntry->SetDataSize(newDataSize); + mCacheEntry->TouchData(); + } + return rv; +} + + +NS_IMETHODIMP +nsCacheEntryDescriptor::SetDataSize(uint32_t dataSize) +{ + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHEENTRYDESCRIPTOR_SETDATASIZE)); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + + // XXX review for signed/unsigned math errors + int32_t deltaSize = dataSize - mCacheEntry->DataSize(); + + nsresult rv; + rv = nsCacheService::OnDataSizeChange(mCacheEntry, deltaSize); + // this had better be NS_OK, this call instance is advisory for memory cache objects + if (NS_SUCCEEDED(rv)) { + // XXX review for signed/unsigned math errors + uint32_t newDataSize = mCacheEntry->DataSize() + deltaSize; + mCacheEntry->SetDataSize(newDataSize); + mCacheEntry->TouchData(); + } else { + NS_WARNING("failed SetDataSize() on memory cache object!"); + } + + return rv; +} + + +NS_IMETHODIMP +nsCacheEntryDescriptor::OpenInputStream(uint32_t offset, nsIInputStream ** result) +{ + NS_ENSURE_ARG_POINTER(result); + + nsInputStreamWrapper* cacheInput = nullptr; + { + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHEENTRYDESCRIPTOR_OPENINPUTSTREAM)); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + if (!mCacheEntry->IsStreamData()) return NS_ERROR_CACHE_DATA_IS_NOT_STREAM; + + // Don't open any new stream when closing descriptor or clearing entries + if (mClosingDescriptor || nsCacheService::GetClearingEntries()) + return NS_ERROR_NOT_AVAILABLE; + + // ensure valid permissions + if (!(mAccessGranted & nsICache::ACCESS_READ)) + return NS_ERROR_CACHE_READ_ACCESS_DENIED; + + const char *val; + val = mCacheEntry->GetMetaDataElement("uncompressed-len"); + if (val) { + cacheInput = new nsDecompressInputStreamWrapper(this, offset); + } else { + cacheInput = new nsInputStreamWrapper(this, offset); + } + if (!cacheInput) return NS_ERROR_OUT_OF_MEMORY; + + mInputWrappers.AppendElement(cacheInput); + } + + NS_ADDREF(*result = cacheInput); + return NS_OK; +} + +NS_IMETHODIMP +nsCacheEntryDescriptor::OpenOutputStream(uint32_t offset, nsIOutputStream ** result) +{ + NS_ENSURE_ARG_POINTER(result); + + nsOutputStreamWrapper* cacheOutput = nullptr; + { + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHEENTRYDESCRIPTOR_OPENOUTPUTSTREAM)); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + if (!mCacheEntry->IsStreamData()) return NS_ERROR_CACHE_DATA_IS_NOT_STREAM; + + // Don't open any new stream when closing descriptor or clearing entries + if (mClosingDescriptor || nsCacheService::GetClearingEntries()) + return NS_ERROR_NOT_AVAILABLE; + + // ensure valid permissions + if (!(mAccessGranted & nsICache::ACCESS_WRITE)) + return NS_ERROR_CACHE_WRITE_ACCESS_DENIED; + + int32_t compressionLevel = nsCacheService::CacheCompressionLevel(); + const char *val; + val = mCacheEntry->GetMetaDataElement("uncompressed-len"); + if ((compressionLevel > 0) && val) { + cacheOutput = new nsCompressOutputStreamWrapper(this, offset); + } else { + // clear compression flag when compression disabled - see bug 715198 + if (val) { + mCacheEntry->SetMetaDataElement("uncompressed-len", nullptr); + } + cacheOutput = new nsOutputStreamWrapper(this, offset); + } + if (!cacheOutput) return NS_ERROR_OUT_OF_MEMORY; + + mOutputWrapper = cacheOutput; + } + + NS_ADDREF(*result = cacheOutput); + return NS_OK; +} + + +NS_IMETHODIMP +nsCacheEntryDescriptor::GetCacheElement(nsISupports ** result) +{ + NS_ENSURE_ARG_POINTER(result); + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHEENTRYDESCRIPTOR_GETCACHEELEMENT)); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + if (mCacheEntry->IsStreamData()) return NS_ERROR_CACHE_DATA_IS_STREAM; + + NS_IF_ADDREF(*result = mCacheEntry->Data()); + return NS_OK; +} + + +NS_IMETHODIMP +nsCacheEntryDescriptor::SetCacheElement(nsISupports * cacheElement) +{ + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHEENTRYDESCRIPTOR_SETCACHEELEMENT)); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + if (mCacheEntry->IsStreamData()) return NS_ERROR_CACHE_DATA_IS_STREAM; + + return nsCacheService::SetCacheElement(mCacheEntry, cacheElement); +} + + +NS_IMETHODIMP +nsCacheEntryDescriptor::GetAccessGranted(nsCacheAccessMode *result) +{ + NS_ENSURE_ARG_POINTER(result); + *result = mAccessGranted; + return NS_OK; +} + + +NS_IMETHODIMP +nsCacheEntryDescriptor::GetStoragePolicy(nsCacheStoragePolicy *result) +{ + NS_ENSURE_ARG_POINTER(result); + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHEENTRYDESCRIPTOR_GETSTORAGEPOLICY)); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + + *result = mCacheEntry->StoragePolicy(); + return NS_OK; +} + + +NS_IMETHODIMP +nsCacheEntryDescriptor::SetStoragePolicy(nsCacheStoragePolicy policy) +{ + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHEENTRYDESCRIPTOR_SETSTORAGEPOLICY)); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + // XXX validate policy against session? + + bool storageEnabled = false; + storageEnabled = nsCacheService::IsStorageEnabledForPolicy_Locked(policy); + if (!storageEnabled) return NS_ERROR_FAILURE; + + // Don't change the storage policy of entries we can't write + if (!(mAccessGranted & nsICache::ACCESS_WRITE)) + return NS_ERROR_NOT_AVAILABLE; + + // Don't allow a cache entry to move from memory-only to anything else + if (mCacheEntry->StoragePolicy() == nsICache::STORE_IN_MEMORY && + policy != nsICache::STORE_IN_MEMORY) + return NS_ERROR_NOT_AVAILABLE; + + mCacheEntry->SetStoragePolicy(policy); + mCacheEntry->MarkEntryDirty(); + return NS_OK; +} + + +NS_IMETHODIMP +nsCacheEntryDescriptor::GetFile(nsIFile ** result) +{ + NS_ENSURE_ARG_POINTER(result); + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHEENTRYDESCRIPTOR_GETFILE)); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + + return nsCacheService::GetFileForEntry(mCacheEntry, result); +} + + +NS_IMETHODIMP +nsCacheEntryDescriptor::GetSecurityInfo(nsISupports ** result) +{ + NS_ENSURE_ARG_POINTER(result); + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHEENTRYDESCRIPTOR_GETSECURITYINFO)); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + + *result = mCacheEntry->SecurityInfo(); + NS_IF_ADDREF(*result); + return NS_OK; +} + + +NS_IMETHODIMP +nsCacheEntryDescriptor::SetSecurityInfo(nsISupports * securityInfo) +{ + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHEENTRYDESCRIPTOR_SETSECURITYINFO)); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + + mCacheEntry->SetSecurityInfo(securityInfo); + mCacheEntry->MarkEntryDirty(); + return NS_OK; +} + + +NS_IMETHODIMP +nsCacheEntryDescriptor::Doom() +{ + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHEENTRYDESCRIPTOR_DOOM)); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + + return nsCacheService::DoomEntry(mCacheEntry); +} + + +NS_IMETHODIMP +nsCacheEntryDescriptor::DoomAndFailPendingRequests(nsresult status) +{ + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHEENTRYDESCRIPTOR_DOOMANDFAILPENDINGREQUESTS)); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + + return NS_ERROR_NOT_IMPLEMENTED; +} + + +NS_IMETHODIMP +nsCacheEntryDescriptor::AsyncDoom(nsICacheListener *listener) +{ + bool asyncDoomPending; + { + mozilla::MutexAutoLock lock(mLock); + asyncDoomPending = mAsyncDoomPending; + mAsyncDoomPending = true; + } + + if (asyncDoomPending) { + // AsyncDoom was already called. Notify listener if it is non-null, + // otherwise just return success. + if (listener) { + nsresult rv = NS_DispatchToCurrentThread( + new nsNotifyDoomListener(listener, NS_ERROR_NOT_AVAILABLE)); + if (NS_SUCCEEDED(rv)) + NS_IF_ADDREF(listener); + return rv; + } + return NS_OK; + } + + nsCOMPtr<nsIRunnable> event = new nsAsyncDoomEvent(this, listener); + return nsCacheService::DispatchToCacheIOThread(event); +} + + +NS_IMETHODIMP +nsCacheEntryDescriptor::MarkValid() +{ + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHEENTRYDESCRIPTOR_MARKVALID)); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + + nsresult rv = nsCacheService::ValidateEntry(mCacheEntry); + return rv; +} + + +NS_IMETHODIMP +nsCacheEntryDescriptor::Close() +{ + RefPtr<nsOutputStreamWrapper> outputWrapper; + nsTArray<RefPtr<nsInputStreamWrapper> > inputWrappers; + + { + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHEENTRYDESCRIPTOR_CLOSE)); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + + // Make sure no other stream can be opened + mClosingDescriptor = true; + outputWrapper = mOutputWrapper; + for (size_t i = 0; i < mInputWrappers.Length(); i++) + inputWrappers.AppendElement(mInputWrappers[i]); + } + + // Call Close() on the streams outside the lock since it might need to call + // methods that grab the cache service lock, e.g. compressed output stream + // when it finalizes the entry + if (outputWrapper) { + if (NS_FAILED(outputWrapper->Close())) { + NS_WARNING("Dooming entry because Close() failed!!!"); + Doom(); + } + outputWrapper = nullptr; + } + + for (uint32_t i = 0 ; i < inputWrappers.Length() ; i++) + inputWrappers[i]->Close(); + + inputWrappers.Clear(); + + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHEENTRYDESCRIPTOR_CLOSE)); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + + // XXX perhaps closing descriptors should clear/sever transports + + // tell nsCacheService we're going away + nsCacheService::CloseDescriptor(this); + NS_ASSERTION(mCacheEntry == nullptr, "mCacheEntry not null"); + + return NS_OK; +} + + +NS_IMETHODIMP +nsCacheEntryDescriptor::GetMetaDataElement(const char *key, char **result) +{ + NS_ENSURE_ARG_POINTER(key); + *result = nullptr; + + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHEENTRYDESCRIPTOR_GETMETADATAELEMENT)); + NS_ENSURE_TRUE(mCacheEntry, NS_ERROR_NOT_AVAILABLE); + + const char *value; + + value = mCacheEntry->GetMetaDataElement(key); + if (!value) return NS_ERROR_NOT_AVAILABLE; + + *result = NS_strdup(value); + if (!*result) return NS_ERROR_OUT_OF_MEMORY; + + return NS_OK; +} + + +NS_IMETHODIMP +nsCacheEntryDescriptor::SetMetaDataElement(const char *key, const char *value) +{ + NS_ENSURE_ARG_POINTER(key); + + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHEENTRYDESCRIPTOR_SETMETADATAELEMENT)); + NS_ENSURE_TRUE(mCacheEntry, NS_ERROR_NOT_AVAILABLE); + + // XXX allow null value, for clearing key? + + nsresult rv = mCacheEntry->SetMetaDataElement(key, value); + if (NS_SUCCEEDED(rv)) + mCacheEntry->TouchMetaData(); + return rv; +} + + +NS_IMETHODIMP +nsCacheEntryDescriptor::VisitMetaData(nsICacheMetaDataVisitor * visitor) +{ + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHEENTRYDESCRIPTOR_VISITMETADATA)); + // XXX check callers, we're calling out of module + NS_ENSURE_ARG_POINTER(visitor); + if (!mCacheEntry) return NS_ERROR_NOT_AVAILABLE; + + return mCacheEntry->VisitMetaDataElements(visitor); +} + + +/****************************************************************************** + * nsCacheInputStream - a wrapper for nsIInputStream keeps the cache entry + * open while referenced. + ******************************************************************************/ + +NS_IMPL_ADDREF(nsCacheEntryDescriptor::nsInputStreamWrapper) +NS_IMETHODIMP_(MozExternalRefCountType) +nsCacheEntryDescriptor::nsInputStreamWrapper::Release() +{ + // Holding a reference to descriptor ensures that cache service won't go + // away. Do not grab cache service lock if there is no descriptor. + RefPtr<nsCacheEntryDescriptor> desc; + + { + mozilla::MutexAutoLock lock(mLock); + desc = mDescriptor; + } + + if (desc) + nsCacheService::Lock(LOCK_TELEM(NSINPUTSTREAMWRAPPER_RELEASE)); + + nsrefcnt count; + NS_PRECONDITION(0 != mRefCnt, "dup release"); + count = --mRefCnt; + NS_LOG_RELEASE(this, count, "nsCacheEntryDescriptor::nsInputStreamWrapper"); + + if (0 == count) { + // don't use desc here since mDescriptor might be already nulled out + if (mDescriptor) { + NS_ASSERTION(mDescriptor->mInputWrappers.Contains(this), + "Wrapper not found in array!"); + mDescriptor->mInputWrappers.RemoveElement(this); + } + + if (desc) + nsCacheService::Unlock(); + + mRefCnt = 1; + delete (this); + return 0; + } + + if (desc) + nsCacheService::Unlock(); + + return count; +} + +NS_INTERFACE_MAP_BEGIN(nsCacheEntryDescriptor::nsInputStreamWrapper) + NS_INTERFACE_MAP_ENTRY(nsIInputStream) + NS_INTERFACE_MAP_ENTRY(nsISupports) +NS_INTERFACE_MAP_END_THREADSAFE + +nsresult nsCacheEntryDescriptor:: +nsInputStreamWrapper::LazyInit() +{ + // Check if we have the descriptor. If not we can't even grab the cache + // lock since it is not ensured that the cache service still exists. + if (!mDescriptor) + return NS_ERROR_NOT_AVAILABLE; + + nsCacheServiceAutoLock lock(LOCK_TELEM(NSINPUTSTREAMWRAPPER_LAZYINIT)); + + nsCacheAccessMode mode; + nsresult rv = mDescriptor->GetAccessGranted(&mode); + if (NS_FAILED(rv)) return rv; + + NS_ENSURE_TRUE(mode & nsICache::ACCESS_READ, NS_ERROR_UNEXPECTED); + + nsCacheEntry* cacheEntry = mDescriptor->CacheEntry(); + if (!cacheEntry) return NS_ERROR_NOT_AVAILABLE; + + rv = nsCacheService::OpenInputStreamForEntry(cacheEntry, mode, + mStartOffset, + getter_AddRefs(mInput)); + + CACHE_LOG_DEBUG(("nsInputStreamWrapper::LazyInit " + "[entry=%p, wrapper=%p, mInput=%p, rv=%d]", + mDescriptor, this, mInput.get(), int(rv))); + + if (NS_FAILED(rv)) return rv; + + mInitialized = true; + return NS_OK; +} + +nsresult nsCacheEntryDescriptor:: +nsInputStreamWrapper::EnsureInit() +{ + if (mInitialized) { + NS_ASSERTION(mDescriptor, "Bad state"); + return NS_OK; + } + + return LazyInit(); +} + +void nsCacheEntryDescriptor:: +nsInputStreamWrapper::CloseInternal() +{ + mLock.AssertCurrentThreadOwns(); + if (!mDescriptor) { + NS_ASSERTION(!mInitialized, "Bad state"); + NS_ASSERTION(!mInput, "Bad state"); + return; + } + + nsCacheServiceAutoLock lock(LOCK_TELEM(NSINPUTSTREAMWRAPPER_CLOSEINTERNAL)); + + if (mDescriptor) { + mDescriptor->mInputWrappers.RemoveElement(this); + nsCacheService::ReleaseObject_Locked(mDescriptor); + mDescriptor = nullptr; + } + mInitialized = false; + mInput = nullptr; +} + +nsresult nsCacheEntryDescriptor:: +nsInputStreamWrapper::Close() +{ + mozilla::MutexAutoLock lock(mLock); + + return Close_Locked(); +} + +nsresult nsCacheEntryDescriptor:: +nsInputStreamWrapper::Close_Locked() +{ + nsresult rv = EnsureInit(); + if (NS_SUCCEEDED(rv)) { + rv = mInput->Close(); + } else { + NS_ASSERTION(!mInput, + "Shouldn't have mInput when EnsureInit() failed"); + } + + // Call CloseInternal() even when EnsureInit() failed, e.g. in case we are + // closing streams with nsCacheService::CloseAllStream() + CloseInternal(); + return rv; +} + +nsresult nsCacheEntryDescriptor:: +nsInputStreamWrapper::Available(uint64_t *avail) +{ + mozilla::MutexAutoLock lock(mLock); + + nsresult rv = EnsureInit(); + if (NS_FAILED(rv)) return rv; + + return mInput->Available(avail); +} + +nsresult nsCacheEntryDescriptor:: +nsInputStreamWrapper::Read(char *buf, uint32_t count, uint32_t *countRead) +{ + mozilla::MutexAutoLock lock(mLock); + + return Read_Locked(buf, count, countRead); +} + +nsresult nsCacheEntryDescriptor:: +nsInputStreamWrapper::Read_Locked(char *buf, uint32_t count, uint32_t *countRead) +{ + nsresult rv = EnsureInit(); + if (NS_SUCCEEDED(rv)) + rv = mInput->Read(buf, count, countRead); + + CACHE_LOG_DEBUG(("nsInputStreamWrapper::Read " + "[entry=%p, wrapper=%p, mInput=%p, rv=%d]", + mDescriptor, this, mInput.get(), rv)); + + return rv; +} + +nsresult nsCacheEntryDescriptor:: +nsInputStreamWrapper::ReadSegments(nsWriteSegmentFun writer, void *closure, + uint32_t count, uint32_t *countRead) +{ + // cache stream not buffered + return NS_ERROR_NOT_IMPLEMENTED; +} + +nsresult nsCacheEntryDescriptor:: +nsInputStreamWrapper::IsNonBlocking(bool *result) +{ + // cache streams will never return NS_BASE_STREAM_WOULD_BLOCK + *result = false; + return NS_OK; +} + + +/****************************************************************************** + * nsDecompressInputStreamWrapper - an input stream wrapper that decompresses + ******************************************************************************/ + +NS_IMPL_ADDREF(nsCacheEntryDescriptor::nsDecompressInputStreamWrapper) +NS_IMETHODIMP_(MozExternalRefCountType) +nsCacheEntryDescriptor::nsDecompressInputStreamWrapper::Release() +{ + // Holding a reference to descriptor ensures that cache service won't go + // away. Do not grab cache service lock if there is no descriptor. + RefPtr<nsCacheEntryDescriptor> desc; + + { + mozilla::MutexAutoLock lock(mLock); + desc = mDescriptor; + } + + if (desc) + nsCacheService::Lock(LOCK_TELEM( + NSDECOMPRESSINPUTSTREAMWRAPPER_RELEASE)); + + nsrefcnt count; + NS_PRECONDITION(0 != mRefCnt, "dup release"); + count = --mRefCnt; + NS_LOG_RELEASE(this, count, + "nsCacheEntryDescriptor::nsDecompressInputStreamWrapper"); + + if (0 == count) { + // don't use desc here since mDescriptor might be already nulled out + if (mDescriptor) { + NS_ASSERTION(mDescriptor->mInputWrappers.Contains(this), + "Wrapper not found in array!"); + mDescriptor->mInputWrappers.RemoveElement(this); + } + + if (desc) + nsCacheService::Unlock(); + + mRefCnt = 1; + delete (this); + return 0; + } + + if (desc) + nsCacheService::Unlock(); + + return count; +} + +NS_INTERFACE_MAP_BEGIN(nsCacheEntryDescriptor::nsDecompressInputStreamWrapper) + NS_INTERFACE_MAP_ENTRY(nsIInputStream) + NS_INTERFACE_MAP_ENTRY(nsISupports) +NS_INTERFACE_MAP_END_THREADSAFE + +NS_IMETHODIMP nsCacheEntryDescriptor:: +nsDecompressInputStreamWrapper::Read(char * buf, + uint32_t count, + uint32_t *countRead) +{ + mozilla::MutexAutoLock lock(mLock); + + int zerr = Z_OK; + nsresult rv = NS_OK; + + if (!mStreamInitialized) { + rv = InitZstream(); + if (NS_FAILED(rv)) { + return rv; + } + } + + mZstream.next_out = (Bytef*)buf; + mZstream.avail_out = count; + + if (mReadBufferLen < count) { + // Allocate a buffer for reading from the input stream. This will + // determine the max number of compressed bytes read from the + // input stream at one time. Making the buffer size proportional + // to the request size is not necessary, but helps minimize the + // number of read requests to the input stream. + uint32_t newBufLen = std::max(count, (uint32_t)kMinDecompressReadBufLen); + unsigned char* newBuf; + newBuf = (unsigned char*)moz_xrealloc(mReadBuffer, + newBufLen); + if (newBuf) { + mReadBuffer = newBuf; + mReadBufferLen = newBufLen; + } + if (!mReadBuffer) { + mReadBufferLen = 0; + return NS_ERROR_OUT_OF_MEMORY; + } + } + + // read and inflate data until the output buffer is full, or + // there is no more data to read + while (NS_SUCCEEDED(rv) && + zerr == Z_OK && + mZstream.avail_out > 0 && + count > 0) { + if (mZstream.avail_in == 0) { + rv = nsInputStreamWrapper::Read_Locked((char*)mReadBuffer, + mReadBufferLen, + &mZstream.avail_in); + if (NS_FAILED(rv) || !mZstream.avail_in) { + break; + } + mZstream.next_in = mReadBuffer; + } + zerr = inflate(&mZstream, Z_NO_FLUSH); + if (zerr == Z_STREAM_END) { + // The compressed data may have been stored in multiple + // chunks/streams. To allow for this case, re-initialize + // the inflate stream and continue decompressing from + // the next byte. + Bytef * saveNextIn = mZstream.next_in; + unsigned int saveAvailIn = mZstream.avail_in; + Bytef * saveNextOut = mZstream.next_out; + unsigned int saveAvailOut = mZstream.avail_out; + inflateReset(&mZstream); + mZstream.next_in = saveNextIn; + mZstream.avail_in = saveAvailIn; + mZstream.next_out = saveNextOut; + mZstream.avail_out = saveAvailOut; + zerr = Z_OK; + } else if (zerr != Z_OK) { + rv = NS_ERROR_INVALID_CONTENT_ENCODING; + } + } + if (NS_SUCCEEDED(rv)) { + *countRead = count - mZstream.avail_out; + } + return rv; +} + +nsresult nsCacheEntryDescriptor:: +nsDecompressInputStreamWrapper::Close() +{ + mozilla::MutexAutoLock lock(mLock); + + if (!mDescriptor) + return NS_ERROR_NOT_AVAILABLE; + + EndZstream(); + if (mReadBuffer) { + free(mReadBuffer); + mReadBuffer = 0; + mReadBufferLen = 0; + } + return nsInputStreamWrapper::Close_Locked(); +} + +nsresult nsCacheEntryDescriptor:: +nsDecompressInputStreamWrapper::InitZstream() +{ + if (!mDescriptor) + return NS_ERROR_NOT_AVAILABLE; + + if (mStreamEnded) + return NS_ERROR_FAILURE; + + // Initialize zlib inflate stream + mZstream.zalloc = Z_NULL; + mZstream.zfree = Z_NULL; + mZstream.opaque = Z_NULL; + mZstream.next_out = Z_NULL; + mZstream.avail_out = 0; + mZstream.next_in = Z_NULL; + mZstream.avail_in = 0; + if (inflateInit(&mZstream) != Z_OK) { + return NS_ERROR_FAILURE; + } + mStreamInitialized = true; + return NS_OK; +} + +nsresult nsCacheEntryDescriptor:: +nsDecompressInputStreamWrapper::EndZstream() +{ + if (mStreamInitialized && !mStreamEnded) { + inflateEnd(&mZstream); + mStreamInitialized = false; + mStreamEnded = true; + } + return NS_OK; +} + + +/****************************************************************************** + * nsOutputStreamWrapper - a wrapper for nsIOutputstream to track the amount of + * data written to a cache entry. + * - also keeps the cache entry open while referenced. + ******************************************************************************/ + +NS_IMPL_ADDREF(nsCacheEntryDescriptor::nsOutputStreamWrapper) +NS_IMETHODIMP_(MozExternalRefCountType) +nsCacheEntryDescriptor::nsOutputStreamWrapper::Release() +{ + // Holding a reference to descriptor ensures that cache service won't go + // away. Do not grab cache service lock if there is no descriptor. + RefPtr<nsCacheEntryDescriptor> desc; + + { + mozilla::MutexAutoLock lock(mLock); + desc = mDescriptor; + } + + if (desc) + nsCacheService::Lock(LOCK_TELEM(NSOUTPUTSTREAMWRAPPER_RELEASE)); + + nsrefcnt count; + NS_PRECONDITION(0 != mRefCnt, "dup release"); + count = --mRefCnt; + NS_LOG_RELEASE(this, count, + "nsCacheEntryDescriptor::nsOutputStreamWrapper"); + + if (0 == count) { + // don't use desc here since mDescriptor might be already nulled out + if (mDescriptor) + mDescriptor->mOutputWrapper = nullptr; + + if (desc) + nsCacheService::Unlock(); + + mRefCnt = 1; + delete (this); + return 0; + } + + if (desc) + nsCacheService::Unlock(); + + return count; +} + +NS_INTERFACE_MAP_BEGIN(nsCacheEntryDescriptor::nsOutputStreamWrapper) + NS_INTERFACE_MAP_ENTRY(nsIOutputStream) + NS_INTERFACE_MAP_ENTRY(nsISupports) +NS_INTERFACE_MAP_END_THREADSAFE + +nsresult nsCacheEntryDescriptor:: +nsOutputStreamWrapper::LazyInit() +{ + // Check if we have the descriptor. If not we can't even grab the cache + // lock since it is not ensured that the cache service still exists. + if (!mDescriptor) + return NS_ERROR_NOT_AVAILABLE; + + nsCacheServiceAutoLock lock(LOCK_TELEM(NSOUTPUTSTREAMWRAPPER_LAZYINIT)); + + nsCacheAccessMode mode; + nsresult rv = mDescriptor->GetAccessGranted(&mode); + if (NS_FAILED(rv)) return rv; + + NS_ENSURE_TRUE(mode & nsICache::ACCESS_WRITE, NS_ERROR_UNEXPECTED); + + nsCacheEntry* cacheEntry = mDescriptor->CacheEntry(); + if (!cacheEntry) return NS_ERROR_NOT_AVAILABLE; + + NS_ASSERTION(mOutput == nullptr, "mOutput set in LazyInit"); + + nsCOMPtr<nsIOutputStream> stream; + rv = nsCacheService::OpenOutputStreamForEntry(cacheEntry, mode, mStartOffset, + getter_AddRefs(stream)); + if (NS_FAILED(rv)) + return rv; + + nsCacheDevice* device = cacheEntry->CacheDevice(); + if (device) { + // the entry has been truncated to mStartOffset bytes, inform device + int32_t size = cacheEntry->DataSize(); + rv = device->OnDataSizeChange(cacheEntry, mStartOffset - size); + if (NS_SUCCEEDED(rv)) + cacheEntry->SetDataSize(mStartOffset); + } else { + rv = NS_ERROR_NOT_AVAILABLE; + } + + // If anything above failed, clean up internal state and get out of here + // (see bug #654926)... + if (NS_FAILED(rv)) { + nsCacheService::ReleaseObject_Locked(stream.forget().take()); + mDescriptor->mOutputWrapper = nullptr; + nsCacheService::ReleaseObject_Locked(mDescriptor); + mDescriptor = nullptr; + mInitialized = false; + return rv; + } + + mOutput = stream; + mInitialized = true; + return NS_OK; +} + +nsresult nsCacheEntryDescriptor:: +nsOutputStreamWrapper::EnsureInit() +{ + if (mInitialized) { + NS_ASSERTION(mDescriptor, "Bad state"); + return NS_OK; + } + + return LazyInit(); +} + +nsresult nsCacheEntryDescriptor:: +nsOutputStreamWrapper::OnWrite(uint32_t count) +{ + if (count > INT32_MAX) return NS_ERROR_UNEXPECTED; + return mDescriptor->RequestDataSizeChange((int32_t)count); +} + +void nsCacheEntryDescriptor:: +nsOutputStreamWrapper::CloseInternal() +{ + mLock.AssertCurrentThreadOwns(); + if (!mDescriptor) { + NS_ASSERTION(!mInitialized, "Bad state"); + NS_ASSERTION(!mOutput, "Bad state"); + return; + } + + nsCacheServiceAutoLock lock(LOCK_TELEM(NSOUTPUTSTREAMWRAPPER_CLOSEINTERNAL)); + + if (mDescriptor) { + mDescriptor->mOutputWrapper = nullptr; + nsCacheService::ReleaseObject_Locked(mDescriptor); + mDescriptor = nullptr; + } + mInitialized = false; + mOutput = nullptr; +} + + +NS_IMETHODIMP nsCacheEntryDescriptor:: +nsOutputStreamWrapper::Close() +{ + mozilla::MutexAutoLock lock(mLock); + + return Close_Locked(); +} + +nsresult nsCacheEntryDescriptor:: +nsOutputStreamWrapper::Close_Locked() +{ + nsresult rv = EnsureInit(); + if (NS_SUCCEEDED(rv)) { + rv = mOutput->Close(); + } else { + NS_ASSERTION(!mOutput, + "Shouldn't have mOutput when EnsureInit() failed"); + } + + // Call CloseInternal() even when EnsureInit() failed, e.g. in case we are + // closing streams with nsCacheService::CloseAllStream() + CloseInternal(); + return rv; +} + +NS_IMETHODIMP nsCacheEntryDescriptor:: +nsOutputStreamWrapper::Flush() +{ + mozilla::MutexAutoLock lock(mLock); + + nsresult rv = EnsureInit(); + if (NS_FAILED(rv)) return rv; + + return mOutput->Flush(); +} + +NS_IMETHODIMP nsCacheEntryDescriptor:: +nsOutputStreamWrapper::Write(const char * buf, + uint32_t count, + uint32_t * result) +{ + mozilla::MutexAutoLock lock(mLock); + return Write_Locked(buf, count, result); +} + +nsresult nsCacheEntryDescriptor:: +nsOutputStreamWrapper::Write_Locked(const char * buf, + uint32_t count, + uint32_t * result) +{ + nsresult rv = EnsureInit(); + if (NS_FAILED(rv)) return rv; + + rv = OnWrite(count); + if (NS_FAILED(rv)) return rv; + + return mOutput->Write(buf, count, result); +} + +NS_IMETHODIMP nsCacheEntryDescriptor:: +nsOutputStreamWrapper::WriteFrom(nsIInputStream * inStr, + uint32_t count, + uint32_t * result) +{ + return NS_ERROR_NOT_IMPLEMENTED; +} + +NS_IMETHODIMP nsCacheEntryDescriptor:: +nsOutputStreamWrapper::WriteSegments(nsReadSegmentFun reader, + void * closure, + uint32_t count, + uint32_t * result) +{ + return NS_ERROR_NOT_IMPLEMENTED; +} + +NS_IMETHODIMP nsCacheEntryDescriptor:: +nsOutputStreamWrapper::IsNonBlocking(bool *result) +{ + // cache streams will never return NS_BASE_STREAM_WOULD_BLOCK + *result = false; + return NS_OK; +} + + +/****************************************************************************** + * nsCompressOutputStreamWrapper - an output stream wrapper that compresses + * data before it is written + ******************************************************************************/ + +NS_IMPL_ADDREF(nsCacheEntryDescriptor::nsCompressOutputStreamWrapper) +NS_IMETHODIMP_(MozExternalRefCountType) +nsCacheEntryDescriptor::nsCompressOutputStreamWrapper::Release() +{ + // Holding a reference to descriptor ensures that cache service won't go + // away. Do not grab cache service lock if there is no descriptor. + RefPtr<nsCacheEntryDescriptor> desc; + + { + mozilla::MutexAutoLock lock(mLock); + desc = mDescriptor; + } + + if (desc) + nsCacheService::Lock(LOCK_TELEM(NSCOMPRESSOUTPUTSTREAMWRAPPER_RELEASE)); + + nsrefcnt count; + NS_PRECONDITION(0 != mRefCnt, "dup release"); + count = --mRefCnt; + NS_LOG_RELEASE(this, count, + "nsCacheEntryDescriptor::nsCompressOutputStreamWrapper"); + + if (0 == count) { + // don't use desc here since mDescriptor might be already nulled out + if (mDescriptor) + mDescriptor->mOutputWrapper = nullptr; + + if (desc) + nsCacheService::Unlock(); + + mRefCnt = 1; + delete (this); + return 0; + } + + if (desc) + nsCacheService::Unlock(); + + return count; +} + +NS_INTERFACE_MAP_BEGIN(nsCacheEntryDescriptor::nsCompressOutputStreamWrapper) + NS_INTERFACE_MAP_ENTRY(nsIOutputStream) + NS_INTERFACE_MAP_ENTRY(nsISupports) +NS_INTERFACE_MAP_END_THREADSAFE + +NS_IMETHODIMP nsCacheEntryDescriptor:: +nsCompressOutputStreamWrapper::Write(const char * buf, + uint32_t count, + uint32_t * result) +{ + mozilla::MutexAutoLock lock(mLock); + + int zerr = Z_OK; + nsresult rv = NS_OK; + + if (!mStreamInitialized) { + rv = InitZstream(); + if (NS_FAILED(rv)) { + return rv; + } + } + + if (!mWriteBuffer) { + // Once allocated, this buffer is referenced by the zlib stream and + // cannot be grown. We use 2x(initial write request) to approximate + // a stream buffer size proportional to request buffers. + mWriteBufferLen = std::max(count*2, (uint32_t)kMinCompressWriteBufLen); + mWriteBuffer = (unsigned char*)moz_xmalloc(mWriteBufferLen); + if (!mWriteBuffer) { + mWriteBufferLen = 0; + return NS_ERROR_OUT_OF_MEMORY; + } + mZstream.next_out = mWriteBuffer; + mZstream.avail_out = mWriteBufferLen; + } + + // Compress (deflate) the requested buffer. Keep going + // until the entire buffer has been deflated. + mZstream.avail_in = count; + mZstream.next_in = (Bytef*)buf; + while (mZstream.avail_in > 0) { + zerr = deflate(&mZstream, Z_NO_FLUSH); + if (zerr == Z_STREAM_ERROR) { + deflateEnd(&mZstream); + mStreamEnded = true; + mStreamInitialized = false; + return NS_ERROR_FAILURE; + } + // Note: Z_BUF_ERROR is non-fatal and sometimes expected here. + + // If the compression stream output buffer is filled, write + // it out to the underlying stream wrapper. + if (mZstream.avail_out == 0) { + rv = WriteBuffer(); + if (NS_FAILED(rv)) { + deflateEnd(&mZstream); + mStreamEnded = true; + mStreamInitialized = false; + return rv; + } + } + } + *result = count; + mUncompressedCount += *result; + return NS_OK; +} + +NS_IMETHODIMP nsCacheEntryDescriptor:: +nsCompressOutputStreamWrapper::Close() +{ + mozilla::MutexAutoLock lock(mLock); + + if (!mDescriptor) + return NS_ERROR_NOT_AVAILABLE; + + nsresult retval = NS_OK; + nsresult rv; + int zerr = 0; + + if (mStreamInitialized) { + // complete compression of any data remaining in the zlib stream + do { + zerr = deflate(&mZstream, Z_FINISH); + rv = WriteBuffer(); + if (NS_FAILED(rv)) + retval = rv; + } while (zerr == Z_OK && rv == NS_OK); + deflateEnd(&mZstream); + mStreamInitialized = false; + } + // Do not allow to initialize stream after calling Close(). + mStreamEnded = true; + + if (mDescriptor->CacheEntry()) { + nsAutoCString uncompressedLenStr; + rv = mDescriptor->GetMetaDataElement("uncompressed-len", + getter_Copies(uncompressedLenStr)); + if (NS_SUCCEEDED(rv)) { + int32_t oldCount = uncompressedLenStr.ToInteger(&rv); + if (NS_SUCCEEDED(rv)) { + mUncompressedCount += oldCount; + } + } + uncompressedLenStr.Adopt(0); + uncompressedLenStr.AppendInt(mUncompressedCount); + rv = mDescriptor->SetMetaDataElement("uncompressed-len", + uncompressedLenStr.get()); + if (NS_FAILED(rv)) + retval = rv; + } + + if (mWriteBuffer) { + free(mWriteBuffer); + mWriteBuffer = 0; + mWriteBufferLen = 0; + } + + rv = nsOutputStreamWrapper::Close_Locked(); + if (NS_FAILED(rv)) + retval = rv; + + return retval; +} + +nsresult nsCacheEntryDescriptor:: +nsCompressOutputStreamWrapper::InitZstream() +{ + if (!mDescriptor) + return NS_ERROR_NOT_AVAILABLE; + + if (mStreamEnded) + return NS_ERROR_FAILURE; + + // Determine compression level: Aggressive compression + // may impact performance on mobile devices, while a + // lower compression level still provides substantial + // space savings for many text streams. + int32_t compressionLevel = nsCacheService::CacheCompressionLevel(); + + // Initialize zlib deflate stream + mZstream.zalloc = Z_NULL; + mZstream.zfree = Z_NULL; + mZstream.opaque = Z_NULL; + if (deflateInit2(&mZstream, compressionLevel, Z_DEFLATED, + MAX_WBITS, 8, Z_DEFAULT_STRATEGY) != Z_OK) { + return NS_ERROR_FAILURE; + } + mZstream.next_in = Z_NULL; + mZstream.avail_in = 0; + + mStreamInitialized = true; + + return NS_OK; +} + +nsresult nsCacheEntryDescriptor:: +nsCompressOutputStreamWrapper::WriteBuffer() +{ + uint32_t bytesToWrite = mWriteBufferLen - mZstream.avail_out; + uint32_t result = 0; + nsresult rv = nsCacheEntryDescriptor::nsOutputStreamWrapper::Write_Locked( + (const char *)mWriteBuffer, bytesToWrite, &result); + mZstream.next_out = mWriteBuffer; + mZstream.avail_out = mWriteBufferLen; + return rv; +} + diff --git a/netwerk/cache/nsCacheEntryDescriptor.h b/netwerk/cache/nsCacheEntryDescriptor.h new file mode 100644 index 000000000..64f900811 --- /dev/null +++ b/netwerk/cache/nsCacheEntryDescriptor.h @@ -0,0 +1,237 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + + +#ifndef _nsCacheEntryDescriptor_h_ +#define _nsCacheEntryDescriptor_h_ + +#include "nsICacheEntryDescriptor.h" +#include "nsCacheEntry.h" +#include "nsIInputStream.h" +#include "nsIOutputStream.h" +#include "nsCacheService.h" +#include "zlib.h" +#include "mozilla/Mutex.h" + +/****************************************************************************** +* nsCacheEntryDescriptor +*******************************************************************************/ +class nsCacheEntryDescriptor final : + public PRCList, + public nsICacheEntryDescriptor +{ +public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSICACHEENTRYDESCRIPTOR + NS_DECL_NSICACHEENTRYINFO + + friend class nsAsyncDoomEvent; + friend class nsCacheService; + + nsCacheEntryDescriptor(nsCacheEntry * entry, nsCacheAccessMode mode); + + /** + * utility method to attempt changing data size of associated entry + */ + nsresult RequestDataSizeChange(int32_t deltaSize); + + /** + * methods callbacks for nsCacheService + */ + nsCacheEntry * CacheEntry(void) { return mCacheEntry; } + bool ClearCacheEntry(void) + { + NS_ASSERTION(mInputWrappers.IsEmpty(), "Bad state"); + NS_ASSERTION(!mOutputWrapper, "Bad state"); + + bool doomEntry = false; + bool asyncDoomPending; + { + mozilla::MutexAutoLock lock(mLock); + asyncDoomPending = mAsyncDoomPending; + } + + if (asyncDoomPending && mCacheEntry) { + doomEntry = true; + mDoomedOnClose = true; + } + mCacheEntry = nullptr; + + return doomEntry; + } + +private: + virtual ~nsCacheEntryDescriptor(); + + /************************************************************************* + * input stream wrapper class - + * + * The input stream wrapper references the descriptor, but the descriptor + * doesn't need any references to the stream wrapper. + *************************************************************************/ + class nsInputStreamWrapper : public nsIInputStream { + friend class nsCacheEntryDescriptor; + + private: + nsCacheEntryDescriptor * mDescriptor; + nsCOMPtr<nsIInputStream> mInput; + uint32_t mStartOffset; + bool mInitialized; + mozilla::Mutex mLock; + public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSIINPUTSTREAM + + nsInputStreamWrapper(nsCacheEntryDescriptor * desc, uint32_t off) + : mDescriptor(desc) + , mStartOffset(off) + , mInitialized(false) + , mLock("nsInputStreamWrapper.mLock") + { + NS_ADDREF(mDescriptor); + } + + private: + virtual ~nsInputStreamWrapper() + { + NS_IF_RELEASE(mDescriptor); + } + + nsresult LazyInit(); + nsresult EnsureInit(); + nsresult Read_Locked(char *buf, uint32_t count, uint32_t *countRead); + nsresult Close_Locked(); + void CloseInternal(); + }; + + + class nsDecompressInputStreamWrapper : public nsInputStreamWrapper { + private: + unsigned char* mReadBuffer; + uint32_t mReadBufferLen; + z_stream mZstream; + bool mStreamInitialized; + bool mStreamEnded; + public: + NS_DECL_ISUPPORTS_INHERITED + + nsDecompressInputStreamWrapper(nsCacheEntryDescriptor * desc, + uint32_t off) + : nsInputStreamWrapper(desc, off) + , mReadBuffer(0) + , mReadBufferLen(0) + , mStreamInitialized(false) + , mStreamEnded(false) + { + } + NS_IMETHOD Read(char* buf, uint32_t count, uint32_t * result) override; + NS_IMETHOD Close() override; + private: + virtual ~nsDecompressInputStreamWrapper() + { + Close(); + } + nsresult InitZstream(); + nsresult EndZstream(); + }; + + + /************************************************************************* + * output stream wrapper class - + * + * The output stream wrapper references the descriptor, but the descriptor + * doesn't need any references to the stream wrapper. + *************************************************************************/ + class nsOutputStreamWrapper : public nsIOutputStream { + friend class nsCacheEntryDescriptor; + + protected: + nsCacheEntryDescriptor * mDescriptor; + nsCOMPtr<nsIOutputStream> mOutput; + uint32_t mStartOffset; + bool mInitialized; + mozilla::Mutex mLock; + public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSIOUTPUTSTREAM + + nsOutputStreamWrapper(nsCacheEntryDescriptor * desc, uint32_t off) + : mDescriptor(desc) + , mStartOffset(off) + , mInitialized(false) + , mLock("nsOutputStreamWrapper.mLock") + { + NS_ADDREF(mDescriptor); // owning ref + } + + private: + virtual ~nsOutputStreamWrapper() + { + Close(); + + NS_ASSERTION(!mOutput, "Bad state"); + NS_ASSERTION(!mDescriptor, "Bad state"); + } + + nsresult LazyInit(); + nsresult EnsureInit(); + nsresult OnWrite(uint32_t count); + nsresult Write_Locked(const char * buf, + uint32_t count, + uint32_t * result); + nsresult Close_Locked(); + void CloseInternal(); + }; + + + class nsCompressOutputStreamWrapper : public nsOutputStreamWrapper { + private: + unsigned char* mWriteBuffer; + uint32_t mWriteBufferLen; + z_stream mZstream; + bool mStreamInitialized; + bool mStreamEnded; + uint32_t mUncompressedCount; + public: + NS_DECL_ISUPPORTS_INHERITED + + nsCompressOutputStreamWrapper(nsCacheEntryDescriptor * desc, + uint32_t off) + : nsOutputStreamWrapper(desc, off) + , mWriteBuffer(0) + , mWriteBufferLen(0) + , mStreamInitialized(false) + , mStreamEnded(false) + , mUncompressedCount(0) + { + } + NS_IMETHOD Write(const char* buf, uint32_t count, uint32_t * result) override; + NS_IMETHOD Close() override; + private: + virtual ~nsCompressOutputStreamWrapper() + { + Close(); + } + nsresult InitZstream(); + nsresult WriteBuffer(); + }; + + private: + /** + * nsCacheEntryDescriptor data members + */ + nsCacheEntry * mCacheEntry; // we are a child of the entry + nsCacheAccessMode mAccessGranted; + nsTArray<nsInputStreamWrapper*> mInputWrappers; + nsOutputStreamWrapper * mOutputWrapper; + mozilla::Mutex mLock; + bool mAsyncDoomPending; + bool mDoomedOnClose; + bool mClosingDescriptor; +}; + + +#endif // _nsCacheEntryDescriptor_h_ diff --git a/netwerk/cache/nsCacheMetaData.cpp b/netwerk/cache/nsCacheMetaData.cpp new file mode 100644 index 000000000..0df867d83 --- /dev/null +++ b/netwerk/cache/nsCacheMetaData.cpp @@ -0,0 +1,162 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsCacheMetaData.h" +#include "nsICacheEntryDescriptor.h" + +const char * +nsCacheMetaData::GetElement(const char * key) +{ + const char * data = mBuffer; + const char * limit = mBuffer + mMetaSize; + + while (data < limit) { + // Point to the value part + const char * value = data + strlen(data) + 1; + MOZ_ASSERT(value < limit, "Cache Metadata corrupted"); + if (strcmp(data, key) == 0) + return value; + + // Skip value part + data = value + strlen(value) + 1; + } + MOZ_ASSERT(data == limit, "Metadata corrupted"); + return nullptr; +} + + +nsresult +nsCacheMetaData::SetElement(const char * key, + const char * value) +{ + const uint32_t keySize = strlen(key) + 1; + char * pos = (char *)GetElement(key); + + if (!value) { + // No value means remove the key/value pair completely, if existing + if (pos) { + uint32_t oldValueSize = strlen(pos) + 1; + uint32_t offset = pos - mBuffer; + uint32_t remainder = mMetaSize - (offset + oldValueSize); + + memmove(pos - keySize, pos + oldValueSize, remainder); + mMetaSize -= keySize + oldValueSize; + } + return NS_OK; + } + + const uint32_t valueSize = strlen(value) + 1; + uint32_t newSize = mMetaSize + valueSize; + if (pos) { + const uint32_t oldValueSize = strlen(pos) + 1; + const uint32_t offset = pos - mBuffer; + const uint32_t remainder = mMetaSize - (offset + oldValueSize); + + // Update the value in place + newSize -= oldValueSize; + nsresult rv = EnsureBuffer(newSize); + NS_ENSURE_SUCCESS(rv, rv); + + // Move the remainder to the right place + pos = mBuffer + offset; + memmove(pos + valueSize, pos + oldValueSize, remainder); + } else { + // allocate new meta data element + newSize += keySize; + nsresult rv = EnsureBuffer(newSize); + NS_ENSURE_SUCCESS(rv, rv); + + // Add after last element + pos = mBuffer + mMetaSize; + memcpy(pos, key, keySize); + pos += keySize; + } + + // Update value + memcpy(pos, value, valueSize); + mMetaSize = newSize; + + return NS_OK; +} + +nsresult +nsCacheMetaData::FlattenMetaData(char * buffer, uint32_t bufSize) +{ + if (mMetaSize > bufSize) { + NS_ERROR("buffer size too small for meta data."); + return NS_ERROR_OUT_OF_MEMORY; + } + + memcpy(buffer, mBuffer, mMetaSize); + return NS_OK; +} + +nsresult +nsCacheMetaData::UnflattenMetaData(const char * data, uint32_t size) +{ + if (data && size) { + // Check if the metadata ends with a zero byte. + if (data[size-1] != '\0') { + NS_ERROR("Cache MetaData is not null terminated"); + return NS_ERROR_ILLEGAL_VALUE; + } + // Check that there are an even number of zero bytes + // to match the pattern { key \0 value \0 } + bool odd = false; + for (uint32_t i = 0; i < size; i++) { + if (data[i] == '\0') + odd = !odd; + } + if (odd) { + NS_ERROR("Cache MetaData is malformed"); + return NS_ERROR_ILLEGAL_VALUE; + } + + nsresult rv = EnsureBuffer(size); + NS_ENSURE_SUCCESS(rv, rv); + + memcpy(mBuffer, data, size); + mMetaSize = size; + } + return NS_OK; +} + +nsresult +nsCacheMetaData::VisitElements(nsICacheMetaDataVisitor * visitor) +{ + const char * data = mBuffer; + const char * limit = mBuffer + mMetaSize; + + while (data < limit) { + const char * key = data; + // Skip key part + data += strlen(data) + 1; + MOZ_ASSERT(data < limit, "Metadata corrupted"); + bool keepGoing; + nsresult rv = visitor->VisitMetaDataElement(key, data, &keepGoing); + if (NS_FAILED(rv) || !keepGoing) + return NS_OK; + + // Skip value part + data += strlen(data) + 1; + } + MOZ_ASSERT(data == limit, "Metadata corrupted"); + return NS_OK; +} + +nsresult +nsCacheMetaData::EnsureBuffer(uint32_t bufSize) +{ + if (mBufferSize < bufSize) { + char * buf = (char *)realloc(mBuffer, bufSize); + if (!buf) { + return NS_ERROR_OUT_OF_MEMORY; + } + mBuffer = buf; + mBufferSize = bufSize; + } + return NS_OK; +} diff --git a/netwerk/cache/nsCacheMetaData.h b/netwerk/cache/nsCacheMetaData.h new file mode 100644 index 000000000..8ac19125a --- /dev/null +++ b/netwerk/cache/nsCacheMetaData.h @@ -0,0 +1,45 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef _nsCacheMetaData_h_ +#define _nsCacheMetaData_h_ + +#include "nspr.h" +#include "nscore.h" + +class nsICacheMetaDataVisitor; + +class nsCacheMetaData { +public: + nsCacheMetaData() : mBuffer(nullptr), mBufferSize(0), mMetaSize(0) { } + + ~nsCacheMetaData() { + mBufferSize = mMetaSize = 0; + free(mBuffer); + mBuffer = nullptr; + } + + const char * GetElement(const char * key); + + nsresult SetElement(const char * key, const char * value); + + uint32_t Size(void) { return mMetaSize; } + + nsresult FlattenMetaData(char * buffer, uint32_t bufSize); + + nsresult UnflattenMetaData(const char * buffer, uint32_t bufSize); + + nsresult VisitElements(nsICacheMetaDataVisitor * visitor); + +private: + nsresult EnsureBuffer(uint32_t size); + + char * mBuffer; + uint32_t mBufferSize; + uint32_t mMetaSize; +}; + +#endif // _nsCacheMetaData_h diff --git a/netwerk/cache/nsCacheRequest.h b/netwerk/cache/nsCacheRequest.h new file mode 100644 index 000000000..6681dec67 --- /dev/null +++ b/netwerk/cache/nsCacheRequest.h @@ -0,0 +1,158 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef _nsCacheRequest_h_ +#define _nsCacheRequest_h_ + +#include "nspr.h" +#include "mozilla/CondVar.h" +#include "mozilla/Mutex.h" +#include "nsCOMPtr.h" +#include "nsICache.h" +#include "nsICacheListener.h" +#include "nsCacheSession.h" +#include "nsCacheService.h" + + +class nsCacheRequest : public PRCList +{ + typedef mozilla::CondVar CondVar; + typedef mozilla::MutexAutoLock MutexAutoLock; + typedef mozilla::Mutex Mutex; + +private: + friend class nsCacheService; + friend class nsCacheEntry; + friend class nsProcessRequestEvent; + + nsCacheRequest( const nsACString & key, + nsICacheListener * listener, + nsCacheAccessMode accessRequested, + bool blockingMode, + nsCacheSession * session) + : mKey(key), + mInfo(0), + mListener(listener), + mLock("nsCacheRequest.mLock"), + mCondVar(mLock, "nsCacheRequest.mCondVar"), + mProfileDir(session->ProfileDir()) + { + MOZ_COUNT_CTOR(nsCacheRequest); + PR_INIT_CLIST(this); + SetAccessRequested(accessRequested); + SetStoragePolicy(session->StoragePolicy()); + if (session->IsStreamBased()) MarkStreamBased(); + if (session->WillDoomEntriesIfExpired()) MarkDoomEntriesIfExpired(); + if (session->IsPrivate()) MarkPrivate(); + if (blockingMode == nsICache::BLOCKING) MarkBlockingMode(); + MarkWaitingForValidation(); + NS_IF_ADDREF(mListener); + } + + ~nsCacheRequest() + { + MOZ_COUNT_DTOR(nsCacheRequest); + NS_ASSERTION(PR_CLIST_IS_EMPTY(this), "request still on a list"); + + if (mListener) + nsCacheService::ReleaseObject_Locked(mListener, mThread); + } + + /** + * Simple Accessors + */ + enum CacheRequestInfo { + eStoragePolicyMask = 0x000000FF, + eStreamBasedMask = 0x00000100, + ePrivateMask = 0x00000200, + eDoomEntriesIfExpiredMask = 0x00001000, + eBlockingModeMask = 0x00010000, + eWaitingForValidationMask = 0x00100000, + eAccessRequestedMask = 0xFF000000 + }; + + void SetAccessRequested(nsCacheAccessMode mode) + { + NS_ASSERTION(mode <= 0xFF, "too many bits in nsCacheAccessMode"); + mInfo &= ~eAccessRequestedMask; + mInfo |= mode << 24; + } + + nsCacheAccessMode AccessRequested() + { + return (nsCacheAccessMode)((mInfo >> 24) & 0xFF); + } + + void MarkStreamBased() { mInfo |= eStreamBasedMask; } + bool IsStreamBased() { return (mInfo & eStreamBasedMask) != 0; } + + + void MarkDoomEntriesIfExpired() { mInfo |= eDoomEntriesIfExpiredMask; } + bool WillDoomEntriesIfExpired() { return (0 != (mInfo & eDoomEntriesIfExpiredMask)); } + + void MarkBlockingMode() { mInfo |= eBlockingModeMask; } + bool IsBlocking() { return (0 != (mInfo & eBlockingModeMask)); } + bool IsNonBlocking() { return !(mInfo & eBlockingModeMask); } + + void SetStoragePolicy(nsCacheStoragePolicy policy) + { + NS_ASSERTION(policy <= 0xFF, "too many bits in nsCacheStoragePolicy"); + mInfo &= ~eStoragePolicyMask; // clear storage policy bits + mInfo |= policy; // or in new bits + } + + nsCacheStoragePolicy StoragePolicy() + { + return (nsCacheStoragePolicy)(mInfo & eStoragePolicyMask); + } + + void MarkPrivate() { mInfo |= ePrivateMask; } + void MarkPublic() { mInfo &= ~ePrivateMask; } + bool IsPrivate() { return (mInfo & ePrivateMask) != 0; } + + void MarkWaitingForValidation() { mInfo |= eWaitingForValidationMask; } + void DoneWaitingForValidation() { mInfo &= ~eWaitingForValidationMask; } + bool WaitingForValidation() + { + return (mInfo & eWaitingForValidationMask) != 0; + } + + nsresult + WaitForValidation(void) + { + if (!WaitingForValidation()) { // flag already cleared + MarkWaitingForValidation(); // set up for next time + return NS_OK; // early exit; + } + { + MutexAutoLock lock(mLock); + while (WaitingForValidation()) { + mCondVar.Wait(); + } + MarkWaitingForValidation(); // set up for next time + } + return NS_OK; + } + + void WakeUp(void) { + DoneWaitingForValidation(); + MutexAutoLock lock(mLock); + mCondVar.Notify(); + } + + /** + * Data members + */ + nsCString mKey; + uint32_t mInfo; + nsICacheListener * mListener; // strong ref + nsCOMPtr<nsIThread> mThread; + Mutex mLock; + CondVar mCondVar; + nsCOMPtr<nsIFile> mProfileDir; +}; + +#endif // _nsCacheRequest_h_ diff --git a/netwerk/cache/nsCacheService.cpp b/netwerk/cache/nsCacheService.cpp new file mode 100644 index 000000000..bab67e104 --- /dev/null +++ b/netwerk/cache/nsCacheService.cpp @@ -0,0 +1,3209 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* vim: set ts=8 sts=4 et sw=4 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/ArrayUtils.h" +#include "mozilla/Attributes.h" +#include "mozilla/Assertions.h" +#include "mozilla/DebugOnly.h" + +#include "necko-config.h" + +#include "nsCache.h" +#include "nsCacheService.h" +#include "nsCacheRequest.h" +#include "nsCacheEntry.h" +#include "nsCacheEntryDescriptor.h" +#include "nsCacheDevice.h" +#include "nsMemoryCacheDevice.h" +#include "nsICacheVisitor.h" +#include "nsDiskCacheDevice.h" +#include "nsDiskCacheDeviceSQL.h" +#include "nsCacheUtils.h" +#include "../cache2/CacheObserver.h" + +#include "nsIObserverService.h" +#include "nsIPrefService.h" +#include "nsIPrefBranch.h" +#include "nsIFile.h" +#include "nsIOService.h" +#include "nsDirectoryServiceDefs.h" +#include "nsAppDirectoryServiceDefs.h" +#include "nsThreadUtils.h" +#include "nsProxyRelease.h" +#include "nsDeleteDir.h" +#include "nsNetCID.h" +#include <math.h> // for log() +#include "mozilla/Services.h" +#include "nsITimer.h" +#include "mozIStorageService.h" + +#include "mozilla/net/NeckoCommon.h" +#include <algorithm> + +using namespace mozilla; +using namespace mozilla::net; + +/****************************************************************************** + * nsCacheProfilePrefObserver + *****************************************************************************/ +#define DISK_CACHE_ENABLE_PREF "browser.cache.disk.enable" +#define DISK_CACHE_DIR_PREF "browser.cache.disk.parent_directory" +#define DISK_CACHE_SMART_SIZE_FIRST_RUN_PREF\ + "browser.cache.disk.smart_size.first_run" +#define DISK_CACHE_SMART_SIZE_ENABLED_PREF \ + "browser.cache.disk.smart_size.enabled" +#define DISK_CACHE_SMART_SIZE_PREF "browser.cache.disk.smart_size_cached_value" +#define DISK_CACHE_CAPACITY_PREF "browser.cache.disk.capacity" +#define DISK_CACHE_MAX_ENTRY_SIZE_PREF "browser.cache.disk.max_entry_size" +#define DISK_CACHE_CAPACITY 256000 + +#define DISK_CACHE_USE_OLD_MAX_SMART_SIZE_PREF \ + "browser.cache.disk.smart_size.use_old_max" + +#define OFFLINE_CACHE_ENABLE_PREF "browser.cache.offline.enable" +#define OFFLINE_CACHE_DIR_PREF "browser.cache.offline.parent_directory" +#define OFFLINE_CACHE_CAPACITY_PREF "browser.cache.offline.capacity" +#define OFFLINE_CACHE_CAPACITY 512000 + +#define MEMORY_CACHE_ENABLE_PREF "browser.cache.memory.enable" +#define MEMORY_CACHE_CAPACITY_PREF "browser.cache.memory.capacity" +#define MEMORY_CACHE_MAX_ENTRY_SIZE_PREF "browser.cache.memory.max_entry_size" + +#define CACHE_COMPRESSION_LEVEL_PREF "browser.cache.compression_level" +#define CACHE_COMPRESSION_LEVEL 1 + +#define SANITIZE_ON_SHUTDOWN_PREF "privacy.sanitize.sanitizeOnShutdown" +#define CLEAR_ON_SHUTDOWN_PREF "privacy.clearOnShutdown.cache" + +static const char * observerList[] = { + "profile-before-change", + "profile-do-change", + NS_XPCOM_SHUTDOWN_OBSERVER_ID, + "last-pb-context-exited", + "suspend_process_notification", + "resume_process_notification" +}; + +static const char * prefList[] = { + DISK_CACHE_ENABLE_PREF, + DISK_CACHE_SMART_SIZE_ENABLED_PREF, + DISK_CACHE_CAPACITY_PREF, + DISK_CACHE_DIR_PREF, + DISK_CACHE_MAX_ENTRY_SIZE_PREF, + DISK_CACHE_USE_OLD_MAX_SMART_SIZE_PREF, + OFFLINE_CACHE_ENABLE_PREF, + OFFLINE_CACHE_CAPACITY_PREF, + OFFLINE_CACHE_DIR_PREF, + MEMORY_CACHE_ENABLE_PREF, + MEMORY_CACHE_CAPACITY_PREF, + MEMORY_CACHE_MAX_ENTRY_SIZE_PREF, + CACHE_COMPRESSION_LEVEL_PREF, + SANITIZE_ON_SHUTDOWN_PREF, + CLEAR_ON_SHUTDOWN_PREF +}; + +// Cache sizes, in KB +const int32_t DEFAULT_CACHE_SIZE = 250 * 1024; // 250 MB +#ifdef ANDROID +const int32_t MAX_CACHE_SIZE = 200 * 1024; // 200 MB +const int32_t OLD_MAX_CACHE_SIZE = 200 * 1024; // 200 MB +#else +const int32_t MAX_CACHE_SIZE = 350 * 1024; // 350 MB +const int32_t OLD_MAX_CACHE_SIZE = 1024 * 1024; // 1 GB +#endif +// Default cache size was 50 MB for many years until FF 4: +const int32_t PRE_GECKO_2_0_DEFAULT_CACHE_SIZE = 50 * 1024; + +class nsCacheProfilePrefObserver : public nsIObserver +{ + virtual ~nsCacheProfilePrefObserver() {} + +public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSIOBSERVER + + nsCacheProfilePrefObserver() + : mHaveProfile(false) + , mDiskCacheEnabled(false) + , mDiskCacheCapacity(0) + , mDiskCacheMaxEntrySize(-1) // -1 means "no limit" + , mSmartSizeEnabled(false) + , mShouldUseOldMaxSmartSize(false) + , mOfflineCacheEnabled(false) + , mOfflineCacheCapacity(0) + , mMemoryCacheEnabled(true) + , mMemoryCacheCapacity(-1) + , mMemoryCacheMaxEntrySize(-1) // -1 means "no limit" + , mCacheCompressionLevel(CACHE_COMPRESSION_LEVEL) + , mSanitizeOnShutdown(false) + , mClearCacheOnShutdown(false) + { + } + + nsresult Install(); + void Remove(); + nsresult ReadPrefs(nsIPrefBranch* branch); + + bool DiskCacheEnabled(); + int32_t DiskCacheCapacity() { return mDiskCacheCapacity; } + void SetDiskCacheCapacity(int32_t); + int32_t DiskCacheMaxEntrySize() { return mDiskCacheMaxEntrySize; } + nsIFile * DiskCacheParentDirectory() { return mDiskCacheParentDirectory; } + bool SmartSizeEnabled() { return mSmartSizeEnabled; } + + bool ShouldUseOldMaxSmartSize() { return mShouldUseOldMaxSmartSize; } + void SetUseNewMaxSmartSize(bool useNew) { mShouldUseOldMaxSmartSize = !useNew; } + + bool OfflineCacheEnabled(); + int32_t OfflineCacheCapacity() { return mOfflineCacheCapacity; } + nsIFile * OfflineCacheParentDirectory() { return mOfflineCacheParentDirectory; } + + bool MemoryCacheEnabled(); + int32_t MemoryCacheCapacity(); + int32_t MemoryCacheMaxEntrySize() { return mMemoryCacheMaxEntrySize; } + + int32_t CacheCompressionLevel(); + + bool SanitizeAtShutdown() { return mSanitizeOnShutdown && mClearCacheOnShutdown; } + + static uint32_t GetSmartCacheSize(const nsAString& cachePath, + uint32_t currentSize, + bool shouldUseOldMaxSmartSize); + + bool PermittedToSmartSize(nsIPrefBranch*, bool firstRun); + +private: + bool mHaveProfile; + + bool mDiskCacheEnabled; + int32_t mDiskCacheCapacity; // in kilobytes + int32_t mDiskCacheMaxEntrySize; // in kilobytes + nsCOMPtr<nsIFile> mDiskCacheParentDirectory; + bool mSmartSizeEnabled; + + bool mShouldUseOldMaxSmartSize; + + bool mOfflineCacheEnabled; + int32_t mOfflineCacheCapacity; // in kilobytes + nsCOMPtr<nsIFile> mOfflineCacheParentDirectory; + + bool mMemoryCacheEnabled; + int32_t mMemoryCacheCapacity; // in kilobytes + int32_t mMemoryCacheMaxEntrySize; // in kilobytes + + int32_t mCacheCompressionLevel; + + bool mSanitizeOnShutdown; + bool mClearCacheOnShutdown; +}; + +NS_IMPL_ISUPPORTS(nsCacheProfilePrefObserver, nsIObserver) + +class nsSetDiskSmartSizeCallback final : public nsITimerCallback +{ + ~nsSetDiskSmartSizeCallback() {} + +public: + NS_DECL_THREADSAFE_ISUPPORTS + + NS_IMETHOD Notify(nsITimer* aTimer) override { + if (nsCacheService::gService) { + nsCacheServiceAutoLock autoLock(LOCK_TELEM(NSSETDISKSMARTSIZECALLBACK_NOTIFY)); + nsCacheService::gService->SetDiskSmartSize_Locked(); + nsCacheService::gService->mSmartSizeTimer = nullptr; + } + return NS_OK; + } +}; + +NS_IMPL_ISUPPORTS(nsSetDiskSmartSizeCallback, nsITimerCallback) + +// Runnable sent to main thread after the cache IO thread calculates available +// disk space, so that there is no race in setting mDiskCacheCapacity. +class nsSetSmartSizeEvent: public Runnable +{ +public: + explicit nsSetSmartSizeEvent(int32_t smartSize) + : mSmartSize(smartSize) {} + + NS_IMETHOD Run() + { + NS_ASSERTION(NS_IsMainThread(), + "Setting smart size data off the main thread"); + + // Main thread may have already called nsCacheService::Shutdown + if (!nsCacheService::IsInitialized()) + return NS_ERROR_NOT_AVAILABLE; + + // Ensure smart sizing wasn't switched off while event was pending. + // It is safe to access the observer without the lock since we are + // on the main thread and the value changes only on the main thread. + if (!nsCacheService::gService->mObserver->SmartSizeEnabled()) + return NS_OK; + + nsCacheService::SetDiskCacheCapacity(mSmartSize); + + nsCOMPtr<nsIPrefBranch> ps = do_GetService(NS_PREFSERVICE_CONTRACTID); + if (!ps || + NS_FAILED(ps->SetIntPref(DISK_CACHE_SMART_SIZE_PREF, mSmartSize))) + NS_WARNING("Failed to set smart size pref"); + + return NS_OK; + } + +private: + int32_t mSmartSize; +}; + + +// Runnable sent from main thread to cacheIO thread +class nsGetSmartSizeEvent: public Runnable +{ +public: + nsGetSmartSizeEvent(const nsAString& cachePath, uint32_t currentSize, + bool shouldUseOldMaxSmartSize) + : mCachePath(cachePath) + , mCurrentSize(currentSize) + , mShouldUseOldMaxSmartSize(shouldUseOldMaxSmartSize) + {} + + // Calculates user's disk space available on a background thread and + // dispatches this value back to the main thread. + NS_IMETHOD Run() override + { + uint32_t size; + size = nsCacheProfilePrefObserver::GetSmartCacheSize(mCachePath, + mCurrentSize, + mShouldUseOldMaxSmartSize); + NS_DispatchToMainThread(new nsSetSmartSizeEvent(size)); + return NS_OK; + } + +private: + nsString mCachePath; + uint32_t mCurrentSize; + bool mShouldUseOldMaxSmartSize; +}; + +class nsBlockOnCacheThreadEvent : public Runnable { +public: + nsBlockOnCacheThreadEvent() + { + } + NS_IMETHOD Run() override + { + nsCacheServiceAutoLock autoLock(LOCK_TELEM(NSBLOCKONCACHETHREADEVENT_RUN)); + CACHE_LOG_DEBUG(("nsBlockOnCacheThreadEvent [%p]\n", this)); + nsCacheService::gService->mNotified = true; + nsCacheService::gService->mCondVar.Notify(); + return NS_OK; + } +}; + + +nsresult +nsCacheProfilePrefObserver::Install() +{ + // install profile-change observer + nsCOMPtr<nsIObserverService> observerService = + mozilla::services::GetObserverService(); + if (!observerService) + return NS_ERROR_FAILURE; + + nsresult rv, rv2 = NS_OK; + for (unsigned int i=0; i<ArrayLength(observerList); i++) { + rv = observerService->AddObserver(this, observerList[i], false); + if (NS_FAILED(rv)) + rv2 = rv; + } + + // install preferences observer + nsCOMPtr<nsIPrefBranch> branch = do_GetService(NS_PREFSERVICE_CONTRACTID); + if (!branch) return NS_ERROR_FAILURE; + + for (unsigned int i=0; i<ArrayLength(prefList); i++) { + rv = branch->AddObserver(prefList[i], this, false); + if (NS_FAILED(rv)) + rv2 = rv; + } + + // Determine if we have a profile already + // Install() is called *after* the profile-after-change notification + // when there is only a single profile, or it is specified on the + // commandline at startup. + // In that case, we detect the presence of a profile by the existence + // of the NS_APP_USER_PROFILE_50_DIR directory. + + nsCOMPtr<nsIFile> directory; + rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, + getter_AddRefs(directory)); + if (NS_SUCCEEDED(rv)) + mHaveProfile = true; + + rv = ReadPrefs(branch); + NS_ENSURE_SUCCESS(rv, rv); + + return rv2; +} + + +void +nsCacheProfilePrefObserver::Remove() +{ + // remove Observer Service observers + nsCOMPtr<nsIObserverService> obs = + mozilla::services::GetObserverService(); + if (obs) { + for (unsigned int i=0; i<ArrayLength(observerList); i++) { + obs->RemoveObserver(this, observerList[i]); + } + } + + // remove Pref Service observers + nsCOMPtr<nsIPrefBranch> prefs = + do_GetService(NS_PREFSERVICE_CONTRACTID); + if (!prefs) + return; + for (unsigned int i=0; i<ArrayLength(prefList); i++) + prefs->RemoveObserver(prefList[i], this); // remove cache pref observers +} + +void +nsCacheProfilePrefObserver::SetDiskCacheCapacity(int32_t capacity) +{ + mDiskCacheCapacity = std::max(0, capacity); +} + + +NS_IMETHODIMP +nsCacheProfilePrefObserver::Observe(nsISupports * subject, + const char * topic, + const char16_t * data_unicode) +{ + nsresult rv; + NS_ConvertUTF16toUTF8 data(data_unicode); + CACHE_LOG_INFO(("Observe [topic=%s data=%s]\n", topic, data.get())); + + if (!nsCacheService::IsInitialized()) { + if (!strcmp("resume_process_notification", topic)) { + // A suspended process has a closed cache, so re-open it here. + nsCacheService::GlobalInstance()->Init(); + } + return NS_OK; + } + + if (!strcmp(NS_XPCOM_SHUTDOWN_OBSERVER_ID, topic)) { + // xpcom going away, shutdown cache service + nsCacheService::GlobalInstance()->Shutdown(); + } else if (!strcmp("profile-before-change", topic)) { + // profile before change + mHaveProfile = false; + + // XXX shutdown devices + nsCacheService::OnProfileShutdown(); + } else if (!strcmp("suspend_process_notification", topic)) { + // A suspended process may never return, so shutdown the cache to reduce + // cache corruption. + nsCacheService::GlobalInstance()->Shutdown(); + } else if (!strcmp("profile-do-change", topic)) { + // profile after change + mHaveProfile = true; + nsCOMPtr<nsIPrefBranch> branch = do_GetService(NS_PREFSERVICE_CONTRACTID); + if (!branch) { + return NS_ERROR_FAILURE; + } + (void)ReadPrefs(branch); + nsCacheService::OnProfileChanged(); + + } else if (!strcmp(NS_PREFBRANCH_PREFCHANGE_TOPIC_ID, topic)) { + + // ignore pref changes until we're done switch profiles + if (!mHaveProfile) + return NS_OK; + + nsCOMPtr<nsIPrefBranch> branch = do_QueryInterface(subject, &rv); + if (NS_FAILED(rv)) + return rv; + + // which preference changed? + if (!strcmp(DISK_CACHE_ENABLE_PREF, data.get())) { + + rv = branch->GetBoolPref(DISK_CACHE_ENABLE_PREF, + &mDiskCacheEnabled); + if (NS_FAILED(rv)) + return rv; + nsCacheService::SetDiskCacheEnabled(DiskCacheEnabled()); + + } else if (!strcmp(DISK_CACHE_CAPACITY_PREF, data.get())) { + + int32_t capacity = 0; + rv = branch->GetIntPref(DISK_CACHE_CAPACITY_PREF, &capacity); + if (NS_FAILED(rv)) + return rv; + mDiskCacheCapacity = std::max(0, capacity); + nsCacheService::SetDiskCacheCapacity(mDiskCacheCapacity); + + // Update the cache capacity when smart sizing is turned on/off + } else if (!strcmp(DISK_CACHE_SMART_SIZE_ENABLED_PREF, data.get())) { + // Is the update because smartsizing was turned on, or off? + rv = branch->GetBoolPref(DISK_CACHE_SMART_SIZE_ENABLED_PREF, + &mSmartSizeEnabled); + if (NS_FAILED(rv)) + return rv; + int32_t newCapacity = 0; + if (mSmartSizeEnabled) { + nsCacheService::SetDiskSmartSize(); + } else { + // Smart sizing switched off: use user specified size + rv = branch->GetIntPref(DISK_CACHE_CAPACITY_PREF, &newCapacity); + if (NS_FAILED(rv)) + return rv; + mDiskCacheCapacity = std::max(0, newCapacity); + nsCacheService::SetDiskCacheCapacity(mDiskCacheCapacity); + } + } else if (!strcmp(DISK_CACHE_USE_OLD_MAX_SMART_SIZE_PREF, data.get())) { + rv = branch->GetBoolPref(DISK_CACHE_USE_OLD_MAX_SMART_SIZE_PREF, + &mShouldUseOldMaxSmartSize); + if (NS_FAILED(rv)) + return rv; + } else if (!strcmp(DISK_CACHE_MAX_ENTRY_SIZE_PREF, data.get())) { + int32_t newMaxSize; + rv = branch->GetIntPref(DISK_CACHE_MAX_ENTRY_SIZE_PREF, + &newMaxSize); + if (NS_FAILED(rv)) + return rv; + + mDiskCacheMaxEntrySize = std::max(-1, newMaxSize); + nsCacheService::SetDiskCacheMaxEntrySize(mDiskCacheMaxEntrySize); + +#if 0 + } else if (!strcmp(DISK_CACHE_DIR_PREF, data.get())) { + // XXX We probaby don't want to respond to this pref except after + // XXX profile changes. Ideally, there should be somekind of user + // XXX notification that the pref change won't take effect until + // XXX the next time the profile changes (browser launch) +#endif + } else + + // which preference changed? + if (!strcmp(OFFLINE_CACHE_ENABLE_PREF, data.get())) { + + rv = branch->GetBoolPref(OFFLINE_CACHE_ENABLE_PREF, + &mOfflineCacheEnabled); + if (NS_FAILED(rv)) return rv; + nsCacheService::SetOfflineCacheEnabled(OfflineCacheEnabled()); + + } else if (!strcmp(OFFLINE_CACHE_CAPACITY_PREF, data.get())) { + + int32_t capacity = 0; + rv = branch->GetIntPref(OFFLINE_CACHE_CAPACITY_PREF, &capacity); + if (NS_FAILED(rv)) return rv; + mOfflineCacheCapacity = std::max(0, capacity); + nsCacheService::SetOfflineCacheCapacity(mOfflineCacheCapacity); +#if 0 + } else if (!strcmp(OFFLINE_CACHE_DIR_PREF, data.get())) { + // XXX We probaby don't want to respond to this pref except after + // XXX profile changes. Ideally, there should be some kind of user + // XXX notification that the pref change won't take effect until + // XXX the next time the profile changes (browser launch) +#endif + } else + + if (!strcmp(MEMORY_CACHE_ENABLE_PREF, data.get())) { + + rv = branch->GetBoolPref(MEMORY_CACHE_ENABLE_PREF, + &mMemoryCacheEnabled); + if (NS_FAILED(rv)) + return rv; + nsCacheService::SetMemoryCache(); + + } else if (!strcmp(MEMORY_CACHE_CAPACITY_PREF, data.get())) { + + mMemoryCacheCapacity = -1; + (void) branch->GetIntPref(MEMORY_CACHE_CAPACITY_PREF, + &mMemoryCacheCapacity); + nsCacheService::SetMemoryCache(); + } else if (!strcmp(MEMORY_CACHE_MAX_ENTRY_SIZE_PREF, data.get())) { + int32_t newMaxSize; + rv = branch->GetIntPref(MEMORY_CACHE_MAX_ENTRY_SIZE_PREF, + &newMaxSize); + if (NS_FAILED(rv)) + return rv; + + mMemoryCacheMaxEntrySize = std::max(-1, newMaxSize); + nsCacheService::SetMemoryCacheMaxEntrySize(mMemoryCacheMaxEntrySize); + } else if (!strcmp(CACHE_COMPRESSION_LEVEL_PREF, data.get())) { + mCacheCompressionLevel = CACHE_COMPRESSION_LEVEL; + (void)branch->GetIntPref(CACHE_COMPRESSION_LEVEL_PREF, + &mCacheCompressionLevel); + mCacheCompressionLevel = std::max(0, mCacheCompressionLevel); + mCacheCompressionLevel = std::min(9, mCacheCompressionLevel); + } else if (!strcmp(SANITIZE_ON_SHUTDOWN_PREF, data.get())) { + rv = branch->GetBoolPref(SANITIZE_ON_SHUTDOWN_PREF, + &mSanitizeOnShutdown); + if (NS_FAILED(rv)) + return rv; + nsCacheService::SetDiskCacheEnabled(DiskCacheEnabled()); + } else if (!strcmp(CLEAR_ON_SHUTDOWN_PREF, data.get())) { + rv = branch->GetBoolPref(CLEAR_ON_SHUTDOWN_PREF, + &mClearCacheOnShutdown); + if (NS_FAILED(rv)) + return rv; + nsCacheService::SetDiskCacheEnabled(DiskCacheEnabled()); + } + } else if (!strcmp("last-pb-context-exited", topic)) { + nsCacheService::LeavePrivateBrowsing(); + } + + return NS_OK; +} + +// Returns default ("smart") size (in KB) of cache, given available disk space +// (also in KB) +static uint32_t +SmartCacheSize(const uint32_t availKB, bool shouldUseOldMaxSmartSize) +{ + uint32_t maxSize = shouldUseOldMaxSmartSize ? OLD_MAX_CACHE_SIZE : MAX_CACHE_SIZE; + + if (availKB > 100 * 1024 * 1024) + return maxSize; // skip computing if we're over 100 GB + + // Grow/shrink in 10 MB units, deliberately, so that in the common case we + // don't shrink cache and evict items every time we startup (it's important + // that we don't slow down startup benchmarks). + uint32_t sz10MBs = 0; + uint32_t avail10MBs = availKB / (1024*10); + + // .5% of space above 25 GB + if (avail10MBs > 2500) { + sz10MBs += static_cast<uint32_t>((avail10MBs - 2500)*.005); + avail10MBs = 2500; + } + // 1% of space between 7GB -> 25 GB + if (avail10MBs > 700) { + sz10MBs += static_cast<uint32_t>((avail10MBs - 700)*.01); + avail10MBs = 700; + } + // 5% of space between 500 MB -> 7 GB + if (avail10MBs > 50) { + sz10MBs += static_cast<uint32_t>((avail10MBs - 50)*.05); + avail10MBs = 50; + } + +#ifdef ANDROID + // On Android, smaller/older devices may have very little storage and + // device owners may be sensitive to storage footprint: Use a smaller + // percentage of available space and a smaller minimum. + + // 20% of space up to 500 MB (10 MB min) + sz10MBs += std::max<uint32_t>(1, static_cast<uint32_t>(avail10MBs * .2)); +#else + // 40% of space up to 500 MB (50 MB min) + sz10MBs += std::max<uint32_t>(5, static_cast<uint32_t>(avail10MBs * .4)); +#endif + + return std::min<uint32_t>(maxSize, sz10MBs * 10 * 1024); +} + + /* Computes our best guess for the default size of the user's disk cache, + * based on the amount of space they have free on their hard drive. + * We use a tiered scheme: the more space available, + * the larger the disk cache will be. However, we do not want + * to enable the disk cache to grow to an unbounded size, so the larger the + * user's available space is, the smaller of a percentage we take. We set a + * lower bound of 50MB and an upper bound of 1GB. + * + *@param: None. + *@return: The size that the user's disk cache should default to, in kBytes. + */ +uint32_t +nsCacheProfilePrefObserver::GetSmartCacheSize(const nsAString& cachePath, + uint32_t currentSize, + bool shouldUseOldMaxSmartSize) +{ + // Check for free space on device where cache directory lives + nsresult rv; + nsCOMPtr<nsIFile> + cacheDirectory (do_CreateInstance(NS_LOCAL_FILE_CONTRACTID, &rv)); + if (NS_FAILED(rv) || !cacheDirectory) + return DEFAULT_CACHE_SIZE; + rv = cacheDirectory->InitWithPath(cachePath); + if (NS_FAILED(rv)) + return DEFAULT_CACHE_SIZE; + int64_t bytesAvailable; + rv = cacheDirectory->GetDiskSpaceAvailable(&bytesAvailable); + if (NS_FAILED(rv)) + return DEFAULT_CACHE_SIZE; + + return SmartCacheSize(static_cast<uint32_t>((bytesAvailable / 1024) + + currentSize), + shouldUseOldMaxSmartSize); +} + +/* Determine if we are permitted to dynamically size the user's disk cache based + * on their disk space available. We may do this so long as the pref + * smart_size.enabled is true. + */ +bool +nsCacheProfilePrefObserver::PermittedToSmartSize(nsIPrefBranch* branch, bool + firstRun) +{ + nsresult rv; + if (firstRun) { + // check if user has set cache size in the past + bool userSet; + rv = branch->PrefHasUserValue(DISK_CACHE_CAPACITY_PREF, &userSet); + if (NS_FAILED(rv)) userSet = true; + if (userSet) { + int32_t oldCapacity; + // If user explicitly set cache size to be smaller than old default + // of 50 MB, then keep user's value. Otherwise use smart sizing. + rv = branch->GetIntPref(DISK_CACHE_CAPACITY_PREF, &oldCapacity); + if (oldCapacity < PRE_GECKO_2_0_DEFAULT_CACHE_SIZE) { + mSmartSizeEnabled = false; + branch->SetBoolPref(DISK_CACHE_SMART_SIZE_ENABLED_PREF, + mSmartSizeEnabled); + return mSmartSizeEnabled; + } + } + // Set manual setting to MAX cache size as starting val for any + // adjustment by user: (bug 559942 comment 65) + int32_t maxSize = mShouldUseOldMaxSmartSize ? OLD_MAX_CACHE_SIZE : MAX_CACHE_SIZE; + branch->SetIntPref(DISK_CACHE_CAPACITY_PREF, maxSize); + } + + rv = branch->GetBoolPref(DISK_CACHE_SMART_SIZE_ENABLED_PREF, + &mSmartSizeEnabled); + if (NS_FAILED(rv)) + mSmartSizeEnabled = false; + return mSmartSizeEnabled; +} + + +nsresult +nsCacheProfilePrefObserver::ReadPrefs(nsIPrefBranch* branch) +{ + nsresult rv = NS_OK; + + // read disk cache device prefs + mDiskCacheEnabled = true; // presume disk cache is enabled + (void) branch->GetBoolPref(DISK_CACHE_ENABLE_PREF, &mDiskCacheEnabled); + + mDiskCacheCapacity = DISK_CACHE_CAPACITY; + (void)branch->GetIntPref(DISK_CACHE_CAPACITY_PREF, &mDiskCacheCapacity); + mDiskCacheCapacity = std::max(0, mDiskCacheCapacity); + + (void) branch->GetIntPref(DISK_CACHE_MAX_ENTRY_SIZE_PREF, + &mDiskCacheMaxEntrySize); + mDiskCacheMaxEntrySize = std::max(-1, mDiskCacheMaxEntrySize); + + (void) branch->GetComplexValue(DISK_CACHE_DIR_PREF, // ignore error + NS_GET_IID(nsIFile), + getter_AddRefs(mDiskCacheParentDirectory)); + + (void) branch->GetBoolPref(DISK_CACHE_USE_OLD_MAX_SMART_SIZE_PREF, + &mShouldUseOldMaxSmartSize); + + if (!mDiskCacheParentDirectory) { + nsCOMPtr<nsIFile> directory; + + // try to get the disk cache parent directory + rv = NS_GetSpecialDirectory(NS_APP_CACHE_PARENT_DIR, + getter_AddRefs(directory)); + if (NS_FAILED(rv)) { + // try to get the profile directory (there may not be a profile yet) + nsCOMPtr<nsIFile> profDir; + NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, + getter_AddRefs(profDir)); + NS_GetSpecialDirectory(NS_APP_USER_PROFILE_LOCAL_50_DIR, + getter_AddRefs(directory)); + if (!directory) + directory = profDir; + else if (profDir) { + nsCacheService::MoveOrRemoveDiskCache(profDir, directory, + "Cache"); + } + } + // use file cache in build tree only if asked, to avoid cache dir litter + if (!directory && PR_GetEnv("NECKO_DEV_ENABLE_DISK_CACHE")) { + rv = NS_GetSpecialDirectory(NS_XPCOM_CURRENT_PROCESS_DIR, + getter_AddRefs(directory)); + } + if (directory) + mDiskCacheParentDirectory = do_QueryInterface(directory, &rv); + } + if (mDiskCacheParentDirectory) { + bool firstSmartSizeRun; + rv = branch->GetBoolPref(DISK_CACHE_SMART_SIZE_FIRST_RUN_PREF, + &firstSmartSizeRun); + if (NS_FAILED(rv)) + firstSmartSizeRun = false; + if (PermittedToSmartSize(branch, firstSmartSizeRun)) { + // Avoid evictions: use previous cache size until smart size event + // updates mDiskCacheCapacity + rv = branch->GetIntPref(firstSmartSizeRun ? + DISK_CACHE_CAPACITY_PREF : + DISK_CACHE_SMART_SIZE_PREF, + &mDiskCacheCapacity); + if (NS_FAILED(rv)) + mDiskCacheCapacity = DEFAULT_CACHE_SIZE; + } + + if (firstSmartSizeRun) { + // It is no longer our first run + rv = branch->SetBoolPref(DISK_CACHE_SMART_SIZE_FIRST_RUN_PREF, + false); + if (NS_FAILED(rv)) + NS_WARNING("Failed setting first_run pref in ReadPrefs."); + } + } + + // read offline cache device prefs + mOfflineCacheEnabled = true; // presume offline cache is enabled + (void) branch->GetBoolPref(OFFLINE_CACHE_ENABLE_PREF, + &mOfflineCacheEnabled); + + mOfflineCacheCapacity = OFFLINE_CACHE_CAPACITY; + (void)branch->GetIntPref(OFFLINE_CACHE_CAPACITY_PREF, + &mOfflineCacheCapacity); + mOfflineCacheCapacity = std::max(0, mOfflineCacheCapacity); + + (void) branch->GetComplexValue(OFFLINE_CACHE_DIR_PREF, // ignore error + NS_GET_IID(nsIFile), + getter_AddRefs(mOfflineCacheParentDirectory)); + + if (!mOfflineCacheParentDirectory) { + nsCOMPtr<nsIFile> directory; + + // try to get the offline cache parent directory + rv = NS_GetSpecialDirectory(NS_APP_CACHE_PARENT_DIR, + getter_AddRefs(directory)); + if (NS_FAILED(rv)) { + // try to get the profile directory (there may not be a profile yet) + nsCOMPtr<nsIFile> profDir; + NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, + getter_AddRefs(profDir)); + NS_GetSpecialDirectory(NS_APP_USER_PROFILE_LOCAL_50_DIR, + getter_AddRefs(directory)); + if (!directory) + directory = profDir; + else if (profDir) { + nsCacheService::MoveOrRemoveDiskCache(profDir, directory, + "OfflineCache"); + } + } +#if DEBUG + if (!directory) { + // use current process directory during development + rv = NS_GetSpecialDirectory(NS_XPCOM_CURRENT_PROCESS_DIR, + getter_AddRefs(directory)); + } +#endif + if (directory) + mOfflineCacheParentDirectory = do_QueryInterface(directory, &rv); + } + + // read memory cache device prefs + (void) branch->GetBoolPref(MEMORY_CACHE_ENABLE_PREF, &mMemoryCacheEnabled); + + mMemoryCacheCapacity = -1; + (void) branch->GetIntPref(MEMORY_CACHE_CAPACITY_PREF, + &mMemoryCacheCapacity); + + (void) branch->GetIntPref(MEMORY_CACHE_MAX_ENTRY_SIZE_PREF, + &mMemoryCacheMaxEntrySize); + mMemoryCacheMaxEntrySize = std::max(-1, mMemoryCacheMaxEntrySize); + + // read cache compression level pref + mCacheCompressionLevel = CACHE_COMPRESSION_LEVEL; + (void)branch->GetIntPref(CACHE_COMPRESSION_LEVEL_PREF, + &mCacheCompressionLevel); + mCacheCompressionLevel = std::max(0, mCacheCompressionLevel); + mCacheCompressionLevel = std::min(9, mCacheCompressionLevel); + + // read cache shutdown sanitization prefs + (void) branch->GetBoolPref(SANITIZE_ON_SHUTDOWN_PREF, + &mSanitizeOnShutdown); + (void) branch->GetBoolPref(CLEAR_ON_SHUTDOWN_PREF, + &mClearCacheOnShutdown); + + return rv; +} + +nsresult +nsCacheService::DispatchToCacheIOThread(nsIRunnable* event) +{ + if (!gService || !gService->mCacheIOThread) return NS_ERROR_NOT_AVAILABLE; + return gService->mCacheIOThread->Dispatch(event, NS_DISPATCH_NORMAL); +} + +nsresult +nsCacheService::SyncWithCacheIOThread() +{ + if (!gService || !gService->mCacheIOThread) return NS_ERROR_NOT_AVAILABLE; + gService->mLock.AssertCurrentThreadOwns(); + + nsCOMPtr<nsIRunnable> event = new nsBlockOnCacheThreadEvent(); + + // dispatch event - it will notify the monitor when it's done + nsresult rv = + gService->mCacheIOThread->Dispatch(event, NS_DISPATCH_NORMAL); + if (NS_FAILED(rv)) { + NS_WARNING("Failed dispatching block-event"); + return NS_ERROR_UNEXPECTED; + } + + // wait until notified, then return + gService->mNotified = false; + while (!gService->mNotified) { + gService->mCondVar.Wait(); + } + + return NS_OK; +} + + +bool +nsCacheProfilePrefObserver::DiskCacheEnabled() +{ + if ((mDiskCacheCapacity == 0) || (!mDiskCacheParentDirectory)) return false; + return mDiskCacheEnabled && (!mSanitizeOnShutdown || !mClearCacheOnShutdown); +} + + +bool +nsCacheProfilePrefObserver::OfflineCacheEnabled() +{ + if ((mOfflineCacheCapacity == 0) || (!mOfflineCacheParentDirectory)) + return false; + + return mOfflineCacheEnabled; +} + + +bool +nsCacheProfilePrefObserver::MemoryCacheEnabled() +{ + if (mMemoryCacheCapacity == 0) return false; + return mMemoryCacheEnabled; +} + + +/** + * MemoryCacheCapacity + * + * If the browser.cache.memory.capacity preference is positive, we use that + * value for the amount of memory available for the cache. + * + * If browser.cache.memory.capacity is zero, the memory cache is disabled. + * + * If browser.cache.memory.capacity is negative or not present, we use a + * formula that grows less than linearly with the amount of system memory, + * with an upper limit on the cache size. No matter how much physical RAM is + * present, the default cache size would not exceed 32 MB. This maximum would + * apply only to systems with more than 4 GB of RAM (e.g. terminal servers) + * + * RAM Cache + * --- ----- + * 32 Mb 2 Mb + * 64 Mb 4 Mb + * 128 Mb 6 Mb + * 256 Mb 10 Mb + * 512 Mb 14 Mb + * 1024 Mb 18 Mb + * 2048 Mb 24 Mb + * 4096 Mb 30 Mb + * + * The equation for this is (for cache size C and memory size K (kbytes)): + * x = log2(K) - 14 + * C = x^2/3 + x + 2/3 + 0.1 (0.1 for rounding) + * if (C > 32) C = 32 + */ + +int32_t +nsCacheProfilePrefObserver::MemoryCacheCapacity() +{ + int32_t capacity = mMemoryCacheCapacity; + if (capacity >= 0) { + CACHE_LOG_DEBUG(("Memory cache capacity forced to %d\n", capacity)); + return capacity; + } + + static uint64_t bytes = PR_GetPhysicalMemorySize(); + CACHE_LOG_DEBUG(("Physical Memory size is %llu\n", bytes)); + + // If getting the physical memory failed, arbitrarily assume + // 32 MB of RAM. We use a low default to have a reasonable + // size on all the devices we support. + if (bytes == 0) + bytes = 32 * 1024 * 1024; + + // Conversion from unsigned int64_t to double doesn't work on all platforms. + // We need to truncate the value at INT64_MAX to make sure we don't + // overflow. + if (bytes > INT64_MAX) + bytes = INT64_MAX; + + uint64_t kbytes = bytes >> 10; + + double kBytesD = double(kbytes); + + double x = log(kBytesD)/log(2.0) - 14; + if (x > 0) { + capacity = (int32_t)(x * x / 3.0 + x + 2.0 / 3 + 0.1); // 0.1 for rounding + if (capacity > 32) + capacity = 32; + capacity *= 1024; + } else { + capacity = 0; + } + + return capacity; +} + +int32_t +nsCacheProfilePrefObserver::CacheCompressionLevel() +{ + return mCacheCompressionLevel; +} + +/****************************************************************************** + * nsProcessRequestEvent + *****************************************************************************/ + +class nsProcessRequestEvent : public Runnable { +public: + explicit nsProcessRequestEvent(nsCacheRequest *aRequest) + { + mRequest = aRequest; + } + + NS_IMETHOD Run() override + { + nsresult rv; + + NS_ASSERTION(mRequest->mListener, + "Sync OpenCacheEntry() posted to background thread!"); + + nsCacheServiceAutoLock lock(LOCK_TELEM(NSPROCESSREQUESTEVENT_RUN)); + rv = nsCacheService::gService->ProcessRequest(mRequest, + false, + nullptr); + + // Don't delete the request if it was queued + if (!(mRequest->IsBlocking() && + rv == NS_ERROR_CACHE_WAIT_FOR_VALIDATION)) + delete mRequest; + + return NS_OK; + } + +protected: + virtual ~nsProcessRequestEvent() {} + +private: + nsCacheRequest *mRequest; +}; + +/****************************************************************************** + * nsDoomEvent + *****************************************************************************/ + +class nsDoomEvent : public Runnable { +public: + nsDoomEvent(nsCacheSession *session, + const nsACString &key, + nsICacheListener *listener) + { + mKey = *session->ClientID(); + mKey.Append(':'); + mKey.Append(key); + mStoragePolicy = session->StoragePolicy(); + mListener = listener; + mThread = do_GetCurrentThread(); + // We addref the listener here and release it in nsNotifyDoomListener + // on the callers thread. If posting of nsNotifyDoomListener event fails + // we leak the listener which is better than releasing it on a wrong + // thread. + NS_IF_ADDREF(mListener); + } + + NS_IMETHOD Run() override + { + nsCacheServiceAutoLock lock; + + bool foundActive = true; + nsresult status = NS_ERROR_NOT_AVAILABLE; + nsCacheEntry *entry; + entry = nsCacheService::gService->mActiveEntries.GetEntry(&mKey); + if (!entry) { + bool collision = false; + foundActive = false; + entry = nsCacheService::gService->SearchCacheDevices(&mKey, + mStoragePolicy, + &collision); + } + + if (entry) { + status = NS_OK; + nsCacheService::gService->DoomEntry_Internal(entry, foundActive); + } + + if (mListener) { + mThread->Dispatch(new nsNotifyDoomListener(mListener, status), + NS_DISPATCH_NORMAL); + // posted event will release the reference on the correct thread + mListener = nullptr; + } + + return NS_OK; + } + +private: + nsCString mKey; + nsCacheStoragePolicy mStoragePolicy; + nsICacheListener *mListener; + nsCOMPtr<nsIThread> mThread; +}; + +/****************************************************************************** + * nsCacheService + *****************************************************************************/ +nsCacheService * nsCacheService::gService = nullptr; + +NS_IMPL_ISUPPORTS(nsCacheService, nsICacheService, nsICacheServiceInternal, + nsIMemoryReporter) + +nsCacheService::nsCacheService() + : mObserver(nullptr), + mLock("nsCacheService.mLock"), + mCondVar(mLock, "nsCacheService.mCondVar"), + mNotified(false), + mTimeStampLock("nsCacheService.mTimeStampLock"), + mInitialized(false), + mClearingEntries(false), + mEnableMemoryDevice(true), + mEnableDiskDevice(true), + mMemoryDevice(nullptr), + mDiskDevice(nullptr), + mOfflineDevice(nullptr), + mTotalEntries(0), + mCacheHits(0), + mCacheMisses(0), + mMaxKeyLength(0), + mMaxDataSize(0), + mMaxMetaSize(0), + mDeactivateFailures(0), + mDeactivatedUnboundEntries(0) +{ + NS_ASSERTION(gService==nullptr, "multiple nsCacheService instances!"); + gService = this; + + // create list of cache devices + PR_INIT_CLIST(&mDoomedEntries); +} + +nsCacheService::~nsCacheService() +{ + if (mInitialized) // Shutdown hasn't been called yet. + (void) Shutdown(); + + if (mObserver) { + mObserver->Remove(); + NS_RELEASE(mObserver); + } + + gService = nullptr; +} + + +nsresult +nsCacheService::Init() +{ + // Thie method must be called on the main thread because mCacheIOThread must + // only be modified on the main thread. + if (!NS_IsMainThread()) { + NS_ERROR("nsCacheService::Init called off the main thread"); + return NS_ERROR_NOT_SAME_THREAD; + } + + NS_ASSERTION(!mInitialized, "nsCacheService already initialized."); + if (mInitialized) + return NS_ERROR_ALREADY_INITIALIZED; + + if (mozilla::net::IsNeckoChild()) { + return NS_ERROR_UNEXPECTED; + } + + nsresult rv; + + mStorageService = do_GetService("@mozilla.org/storage/service;1", &rv); + NS_ENSURE_SUCCESS(rv, rv); + + rv = NS_NewNamedThread("Cache I/O", + getter_AddRefs(mCacheIOThread)); + if (NS_FAILED(rv)) { + NS_RUNTIMEABORT("Can't create cache IO thread"); + } + + rv = nsDeleteDir::Init(); + if (NS_FAILED(rv)) { + NS_WARNING("Can't initialize nsDeleteDir"); + } + + // initialize hashtable for active cache entries + mActiveEntries.Init(); + + // create profile/preference observer + if (!mObserver) { + mObserver = new nsCacheProfilePrefObserver(); + NS_ADDREF(mObserver); + mObserver->Install(); + } + + mEnableDiskDevice = mObserver->DiskCacheEnabled(); + mEnableOfflineDevice = mObserver->OfflineCacheEnabled(); + mEnableMemoryDevice = mObserver->MemoryCacheEnabled(); + + RegisterWeakMemoryReporter(this); + + mInitialized = true; + return NS_OK; +} + +void +nsCacheService::Shutdown() +{ + // This method must be called on the main thread because mCacheIOThread must + // only be modified on the main thread. + if (!NS_IsMainThread()) { + NS_RUNTIMEABORT("nsCacheService::Shutdown called off the main thread"); + } + + nsCOMPtr<nsIThread> cacheIOThread; + Telemetry::AutoTimer<Telemetry::NETWORK_DISK_CACHE_SHUTDOWN> totalTimer; + + bool shouldSanitize = false; + nsCOMPtr<nsIFile> parentDir; + + { + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHESERVICE_SHUTDOWN)); + NS_ASSERTION(mInitialized, + "can't shutdown nsCacheService unless it has been initialized."); + if (!mInitialized) + return; + + mClearingEntries = true; + DoomActiveEntries(nullptr); + } + + CloseAllStreams(); + + UnregisterWeakMemoryReporter(this); + + { + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHESERVICE_SHUTDOWN)); + NS_ASSERTION(mInitialized, "Bad state"); + + mInitialized = false; + + // Clear entries + ClearDoomList(); + + if (mSmartSizeTimer) { + mSmartSizeTimer->Cancel(); + mSmartSizeTimer = nullptr; + } + + // Make sure to wait for any pending cache-operations before + // proceeding with destructive actions (bug #620660) + (void) SyncWithCacheIOThread(); + mActiveEntries.Shutdown(); + + // obtain the disk cache directory in case we need to sanitize it + parentDir = mObserver->DiskCacheParentDirectory(); + shouldSanitize = mObserver->SanitizeAtShutdown(); + + // deallocate memory and disk caches + delete mMemoryDevice; + mMemoryDevice = nullptr; + + delete mDiskDevice; + mDiskDevice = nullptr; + + if (mOfflineDevice) + mOfflineDevice->Shutdown(); + + NS_IF_RELEASE(mOfflineDevice); + + for (auto iter = mCustomOfflineDevices.Iter(); + !iter.Done(); iter.Next()) { + iter.Data()->Shutdown(); + iter.Remove(); + } + + LogCacheStatistics(); + + mClearingEntries = false; + mCacheIOThread.swap(cacheIOThread); + } + + if (cacheIOThread) + nsShutdownThread::BlockingShutdown(cacheIOThread); + + if (shouldSanitize) { + nsresult rv = parentDir->AppendNative(NS_LITERAL_CSTRING("Cache")); + if (NS_SUCCEEDED(rv)) { + bool exists; + if (NS_SUCCEEDED(parentDir->Exists(&exists)) && exists) + nsDeleteDir::DeleteDir(parentDir, false); + } + Telemetry::AutoTimer<Telemetry::NETWORK_DISK_CACHE_SHUTDOWN_CLEAR_PRIVATE> timer; + nsDeleteDir::Shutdown(shouldSanitize); + } else { + Telemetry::AutoTimer<Telemetry::NETWORK_DISK_CACHE_DELETEDIR_SHUTDOWN> timer; + nsDeleteDir::Shutdown(shouldSanitize); + } +} + + +nsresult +nsCacheService::Create(nsISupports* aOuter, const nsIID& aIID, void* *aResult) +{ + nsresult rv; + + if (aOuter != nullptr) + return NS_ERROR_NO_AGGREGATION; + + nsCacheService * cacheService = new nsCacheService(); + if (cacheService == nullptr) + return NS_ERROR_OUT_OF_MEMORY; + + NS_ADDREF(cacheService); + rv = cacheService->Init(); + if (NS_SUCCEEDED(rv)) { + rv = cacheService->QueryInterface(aIID, aResult); + } + NS_RELEASE(cacheService); + return rv; +} + + +NS_IMETHODIMP +nsCacheService::CreateSession(const char * clientID, + nsCacheStoragePolicy storagePolicy, + bool streamBased, + nsICacheSession **result) +{ + *result = nullptr; + + if (net::CacheObserver::UseNewCache()) + return NS_ERROR_NOT_IMPLEMENTED; + + return CreateSessionInternal(clientID, storagePolicy, streamBased, result); +} + +nsresult +nsCacheService::CreateSessionInternal(const char * clientID, + nsCacheStoragePolicy storagePolicy, + bool streamBased, + nsICacheSession **result) +{ + RefPtr<nsCacheSession> session = + new nsCacheSession(clientID, storagePolicy, streamBased); + session.forget(result); + + return NS_OK; +} + + +nsresult +nsCacheService::EvictEntriesForSession(nsCacheSession * session) +{ + NS_ASSERTION(gService, "nsCacheService::gService is null."); + return gService->EvictEntriesForClient(session->ClientID()->get(), + session->StoragePolicy()); +} + +namespace { + +class EvictionNotifierRunnable : public Runnable +{ +public: + explicit EvictionNotifierRunnable(nsISupports* aSubject) + : mSubject(aSubject) + { } + + NS_DECL_NSIRUNNABLE + +private: + nsCOMPtr<nsISupports> mSubject; +}; + +NS_IMETHODIMP +EvictionNotifierRunnable::Run() +{ + nsCOMPtr<nsIObserverService> obsSvc = + mozilla::services::GetObserverService(); + if (obsSvc) { + obsSvc->NotifyObservers(mSubject, + NS_CACHESERVICE_EMPTYCACHE_TOPIC_ID, + nullptr); + } + return NS_OK; +} + +} // namespace + +nsresult +nsCacheService::EvictEntriesForClient(const char * clientID, + nsCacheStoragePolicy storagePolicy) +{ + RefPtr<EvictionNotifierRunnable> r = + new EvictionNotifierRunnable(NS_ISUPPORTS_CAST(nsICacheService*, this)); + NS_DispatchToMainThread(r); + + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHESERVICE_EVICTENTRIESFORCLIENT)); + nsresult res = NS_OK; + + if (storagePolicy == nsICache::STORE_ANYWHERE || + storagePolicy == nsICache::STORE_ON_DISK) { + + if (mEnableDiskDevice) { + nsresult rv = NS_OK; + if (!mDiskDevice) + rv = CreateDiskDevice(); + if (mDiskDevice) + rv = mDiskDevice->EvictEntries(clientID); + if (NS_FAILED(rv)) + res = rv; + } + } + + // Only clear the offline cache if it has been specifically asked for. + if (storagePolicy == nsICache::STORE_OFFLINE) { + if (mEnableOfflineDevice) { + nsresult rv = NS_OK; + if (!mOfflineDevice) + rv = CreateOfflineDevice(); + if (mOfflineDevice) + rv = mOfflineDevice->EvictEntries(clientID); + if (NS_FAILED(rv)) + res = rv; + } + } + + if (storagePolicy == nsICache::STORE_ANYWHERE || + storagePolicy == nsICache::STORE_IN_MEMORY) { + // If there is no memory device, there is no need to evict it... + if (mMemoryDevice) { + nsresult rv = mMemoryDevice->EvictEntries(clientID); + if (NS_FAILED(rv)) + res = rv; + } + } + + return res; +} + + +nsresult +nsCacheService::IsStorageEnabledForPolicy(nsCacheStoragePolicy storagePolicy, + bool * result) +{ + if (gService == nullptr) return NS_ERROR_NOT_AVAILABLE; + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHESERVICE_ISSTORAGEENABLEDFORPOLICY)); + + *result = gService->IsStorageEnabledForPolicy_Locked(storagePolicy); + return NS_OK; +} + + +nsresult +nsCacheService::DoomEntry(nsCacheSession *session, + const nsACString &key, + nsICacheListener *listener) +{ + CACHE_LOG_DEBUG(("Dooming entry for session %p, key %s\n", + session, PromiseFlatCString(key).get())); + if (!gService || !gService->mInitialized) + return NS_ERROR_NOT_INITIALIZED; + + return DispatchToCacheIOThread(new nsDoomEvent(session, key, listener)); +} + + +bool +nsCacheService::IsStorageEnabledForPolicy_Locked(nsCacheStoragePolicy storagePolicy) +{ + if (gService->mEnableMemoryDevice && + (storagePolicy == nsICache::STORE_ANYWHERE || + storagePolicy == nsICache::STORE_IN_MEMORY)) { + return true; + } + if (gService->mEnableDiskDevice && + (storagePolicy == nsICache::STORE_ANYWHERE || + storagePolicy == nsICache::STORE_ON_DISK)) { + return true; + } + if (gService->mEnableOfflineDevice && + storagePolicy == nsICache::STORE_OFFLINE) { + return true; + } + + return false; +} + +NS_IMETHODIMP nsCacheService::VisitEntries(nsICacheVisitor *visitor) +{ + if (net::CacheObserver::UseNewCache()) + return NS_ERROR_NOT_IMPLEMENTED; + + return VisitEntriesInternal(visitor); +} + +nsresult nsCacheService::VisitEntriesInternal(nsICacheVisitor *visitor) +{ + NS_ENSURE_ARG_POINTER(visitor); + + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHESERVICE_VISITENTRIES)); + + if (!(mEnableDiskDevice || mEnableMemoryDevice)) + return NS_ERROR_NOT_AVAILABLE; + + // XXX record the fact that a visitation is in progress, + // XXX i.e. keep list of visitors in progress. + + nsresult rv = NS_OK; + // If there is no memory device, there are then also no entries to visit... + if (mMemoryDevice) { + rv = mMemoryDevice->Visit(visitor); + if (NS_FAILED(rv)) return rv; + } + + if (mEnableDiskDevice) { + if (!mDiskDevice) { + rv = CreateDiskDevice(); + if (NS_FAILED(rv)) return rv; + } + rv = mDiskDevice->Visit(visitor); + if (NS_FAILED(rv)) return rv; + } + + if (mEnableOfflineDevice) { + if (!mOfflineDevice) { + rv = CreateOfflineDevice(); + if (NS_FAILED(rv)) return rv; + } + rv = mOfflineDevice->Visit(visitor); + if (NS_FAILED(rv)) return rv; + } + + // XXX notify any shutdown process that visitation is complete for THIS visitor. + // XXX keep queue of visitors + + return NS_OK; +} + +void nsCacheService::FireClearNetworkCacheStoredAnywhereNotification() +{ + MOZ_ASSERT(NS_IsMainThread()); + nsCOMPtr<nsIObserverService> obsvc = mozilla::services::GetObserverService(); + if (obsvc) { + obsvc->NotifyObservers(nullptr, + "network-clear-cache-stored-anywhere", + nullptr); + } +} + +NS_IMETHODIMP nsCacheService::EvictEntries(nsCacheStoragePolicy storagePolicy) +{ + if (net::CacheObserver::UseNewCache()) + return NS_ERROR_NOT_IMPLEMENTED; + + return EvictEntriesInternal(storagePolicy); +} + +nsresult nsCacheService::EvictEntriesInternal(nsCacheStoragePolicy storagePolicy) +{ + if (storagePolicy == nsICache::STORE_ANYWHERE) { + // if not called on main thread, dispatch the notification to the main thread to notify observers + if (!NS_IsMainThread()) { + nsCOMPtr<nsIRunnable> event = NewRunnableMethod(this, + &nsCacheService::FireClearNetworkCacheStoredAnywhereNotification); + NS_DispatchToMainThread(event); + } else { + // else you're already on main thread - notify observers + FireClearNetworkCacheStoredAnywhereNotification(); + } + } + return EvictEntriesForClient(nullptr, storagePolicy); +} + +NS_IMETHODIMP nsCacheService::GetCacheIOTarget(nsIEventTarget * *aCacheIOTarget) +{ + NS_ENSURE_ARG_POINTER(aCacheIOTarget); + + // Because mCacheIOThread can only be changed on the main thread, it can be + // read from the main thread without the lock. This is useful to prevent + // blocking the main thread on other cache operations. + if (!NS_IsMainThread()) { + Lock(LOCK_TELEM(NSCACHESERVICE_GETCACHEIOTARGET)); + } + + nsresult rv; + if (mCacheIOThread) { + NS_ADDREF(*aCacheIOTarget = mCacheIOThread); + rv = NS_OK; + } else { + *aCacheIOTarget = nullptr; + rv = NS_ERROR_NOT_AVAILABLE; + } + + if (!NS_IsMainThread()) { + Unlock(); + } + + return rv; +} + +NS_IMETHODIMP nsCacheService::GetLockHeldTime(double *aLockHeldTime) +{ + MutexAutoLock lock(mTimeStampLock); + + if (mLockAcquiredTimeStamp.IsNull()) { + *aLockHeldTime = 0.0; + } + else { + *aLockHeldTime = + (TimeStamp::Now() - mLockAcquiredTimeStamp).ToMilliseconds(); + } + + return NS_OK; +} + +/** + * Internal Methods + */ +nsresult +nsCacheService::CreateDiskDevice() +{ + if (!mInitialized) return NS_ERROR_NOT_AVAILABLE; + if (!mEnableDiskDevice) return NS_ERROR_NOT_AVAILABLE; + if (mDiskDevice) return NS_OK; + + mDiskDevice = new nsDiskCacheDevice; + if (!mDiskDevice) return NS_ERROR_OUT_OF_MEMORY; + + // set the preferences + mDiskDevice->SetCacheParentDirectory(mObserver->DiskCacheParentDirectory()); + mDiskDevice->SetCapacity(mObserver->DiskCacheCapacity()); + mDiskDevice->SetMaxEntrySize(mObserver->DiskCacheMaxEntrySize()); + + nsresult rv = mDiskDevice->Init(); + if (NS_FAILED(rv)) { +#if DEBUG + printf("###\n"); + printf("### mDiskDevice->Init() failed (0x%.8x)\n", + static_cast<uint32_t>(rv)); + printf("### - disabling disk cache for this session.\n"); + printf("###\n"); +#endif + mEnableDiskDevice = false; + delete mDiskDevice; + mDiskDevice = nullptr; + return rv; + } + + NS_ASSERTION(!mSmartSizeTimer, "Smartsize timer was already fired!"); + + // Disk device is usually created during the startup. Delay smart size + // calculation to avoid possible massive IO caused by eviction of entries + // in case the new smart size is smaller than current cache usage. + mSmartSizeTimer = do_CreateInstance("@mozilla.org/timer;1", &rv); + if (NS_SUCCEEDED(rv)) { + rv = mSmartSizeTimer->InitWithCallback(new nsSetDiskSmartSizeCallback(), + 1000*60*3, + nsITimer::TYPE_ONE_SHOT); + if (NS_FAILED(rv)) { + NS_WARNING("Failed to post smart size timer"); + mSmartSizeTimer = nullptr; + } + } else { + NS_WARNING("Can't create smart size timer"); + } + // Ignore state of the timer and return success since the purpose of the + // method (create the disk-device) has been fulfilled + + return NS_OK; +} + +// Runnable sent from cache thread to main thread +class nsDisableOldMaxSmartSizePrefEvent: public Runnable +{ +public: + nsDisableOldMaxSmartSizePrefEvent() {} + + NS_IMETHOD Run() override + { + // Main thread may have already called nsCacheService::Shutdown + if (!nsCacheService::IsInitialized()) + return NS_ERROR_NOT_AVAILABLE; + + nsCOMPtr<nsIPrefBranch> branch = do_GetService(NS_PREFSERVICE_CONTRACTID); + if (!branch) { + return NS_ERROR_NOT_AVAILABLE; + } + + nsresult rv = branch->SetBoolPref(DISK_CACHE_USE_OLD_MAX_SMART_SIZE_PREF, false); + if (NS_FAILED(rv)) { + NS_WARNING("Failed to disable old max smart size"); + return rv; + } + + // It is safe to call SetDiskSmartSize_Locked() without holding the lock + // when we are on main thread and nsCacheService is initialized. + nsCacheService::gService->SetDiskSmartSize_Locked(); + + if (nsCacheService::gService->mObserver->PermittedToSmartSize(branch, false)) { + rv = branch->SetIntPref(DISK_CACHE_CAPACITY_PREF, MAX_CACHE_SIZE); + if (NS_FAILED(rv)) { + NS_WARNING("Failed to set cache capacity pref"); + } + } + + return NS_OK; + } +}; + +void +nsCacheService::MarkStartingFresh() +{ + if (!gService || !gService->mObserver->ShouldUseOldMaxSmartSize()) { + // Already using new max, nothing to do here + return; + } + + gService->mObserver->SetUseNewMaxSmartSize(true); + + // We always dispatch an event here because we don't want to deal with lock + // reentrance issues. + NS_DispatchToMainThread(new nsDisableOldMaxSmartSizePrefEvent()); +} + +nsresult +nsCacheService::GetOfflineDevice(nsOfflineCacheDevice **aDevice) +{ + if (!mOfflineDevice) { + nsresult rv = CreateOfflineDevice(); + NS_ENSURE_SUCCESS(rv, rv); + } + + NS_ADDREF(*aDevice = mOfflineDevice); + return NS_OK; +} + +nsresult +nsCacheService::GetCustomOfflineDevice(nsIFile *aProfileDir, + int32_t aQuota, + nsOfflineCacheDevice **aDevice) +{ + nsresult rv; + + nsAutoString profilePath; + rv = aProfileDir->GetPath(profilePath); + NS_ENSURE_SUCCESS(rv, rv); + + if (!mCustomOfflineDevices.Get(profilePath, aDevice)) { + rv = CreateCustomOfflineDevice(aProfileDir, aQuota, aDevice); + NS_ENSURE_SUCCESS(rv, rv); + + (*aDevice)->SetAutoShutdown(); + mCustomOfflineDevices.Put(profilePath, *aDevice); + } + + return NS_OK; +} + +nsresult +nsCacheService::CreateOfflineDevice() +{ + CACHE_LOG_INFO(("Creating default offline device")); + + if (mOfflineDevice) return NS_OK; + if (!nsCacheService::IsInitialized()) { + return NS_ERROR_NOT_AVAILABLE; + } + + nsresult rv = CreateCustomOfflineDevice( + mObserver->OfflineCacheParentDirectory(), + mObserver->OfflineCacheCapacity(), + &mOfflineDevice); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult +nsCacheService::CreateCustomOfflineDevice(nsIFile *aProfileDir, + int32_t aQuota, + nsOfflineCacheDevice **aDevice) +{ + NS_ENSURE_ARG(aProfileDir); + + if (MOZ_LOG_TEST(gCacheLog, LogLevel::Info)) { + nsAutoCString profilePath; + aProfileDir->GetNativePath(profilePath); + CACHE_LOG_INFO(("Creating custom offline device, %s, %d", + profilePath.BeginReading(), aQuota)); + } + + if (!mInitialized) return NS_ERROR_NOT_AVAILABLE; + if (!mEnableOfflineDevice) return NS_ERROR_NOT_AVAILABLE; + + *aDevice = new nsOfflineCacheDevice; + + NS_ADDREF(*aDevice); + + // set the preferences + (*aDevice)->SetCacheParentDirectory(aProfileDir); + (*aDevice)->SetCapacity(aQuota); + + nsresult rv = (*aDevice)->InitWithSqlite(mStorageService); + if (NS_FAILED(rv)) { + CACHE_LOG_DEBUG(("OfflineDevice->InitWithSqlite() failed (0x%.8x)\n", rv)); + CACHE_LOG_DEBUG((" - disabling offline cache for this session.\n")); + + NS_RELEASE(*aDevice); + } + return rv; +} + +nsresult +nsCacheService::CreateMemoryDevice() +{ + if (!mInitialized) return NS_ERROR_NOT_AVAILABLE; + if (!mEnableMemoryDevice) return NS_ERROR_NOT_AVAILABLE; + if (mMemoryDevice) return NS_OK; + + mMemoryDevice = new nsMemoryCacheDevice; + if (!mMemoryDevice) return NS_ERROR_OUT_OF_MEMORY; + + // set preference + int32_t capacity = mObserver->MemoryCacheCapacity(); + CACHE_LOG_DEBUG(("Creating memory device with capacity %d\n", capacity)); + mMemoryDevice->SetCapacity(capacity); + mMemoryDevice->SetMaxEntrySize(mObserver->MemoryCacheMaxEntrySize()); + + nsresult rv = mMemoryDevice->Init(); + if (NS_FAILED(rv)) { + NS_WARNING("Initialization of Memory Cache failed."); + delete mMemoryDevice; + mMemoryDevice = nullptr; + } + + return rv; +} + +nsresult +nsCacheService::RemoveCustomOfflineDevice(nsOfflineCacheDevice *aDevice) +{ + nsCOMPtr<nsIFile> profileDir = aDevice->BaseDirectory(); + if (!profileDir) + return NS_ERROR_UNEXPECTED; + + nsAutoString profilePath; + nsresult rv = profileDir->GetPath(profilePath); + NS_ENSURE_SUCCESS(rv, rv); + + mCustomOfflineDevices.Remove(profilePath); + return NS_OK; +} + +nsresult +nsCacheService::CreateRequest(nsCacheSession * session, + const nsACString & clientKey, + nsCacheAccessMode accessRequested, + bool blockingMode, + nsICacheListener * listener, + nsCacheRequest ** request) +{ + NS_ASSERTION(request, "CreateRequest: request is null"); + + nsAutoCString key(*session->ClientID()); + key.Append(':'); + key.Append(clientKey); + + if (mMaxKeyLength < key.Length()) mMaxKeyLength = key.Length(); + + // create request + *request = new nsCacheRequest(key, listener, accessRequested, + blockingMode, session); + + if (!listener) return NS_OK; // we're sync, we're done. + + // get the request's thread + (*request)->mThread = do_GetCurrentThread(); + + return NS_OK; +} + + +class nsCacheListenerEvent : public Runnable +{ +public: + nsCacheListenerEvent(nsICacheListener *listener, + nsICacheEntryDescriptor *descriptor, + nsCacheAccessMode accessGranted, + nsresult status) + : mListener(listener) // transfers reference + , mDescriptor(descriptor) // transfers reference (may be null) + , mAccessGranted(accessGranted) + , mStatus(status) + {} + + NS_IMETHOD Run() override + { + mListener->OnCacheEntryAvailable(mDescriptor, mAccessGranted, mStatus); + + NS_RELEASE(mListener); + NS_IF_RELEASE(mDescriptor); + return NS_OK; + } + +private: + // We explicitly leak mListener or mDescriptor if Run is not called + // because otherwise we cannot guarantee that they are destroyed on + // the right thread. + + nsICacheListener *mListener; + nsICacheEntryDescriptor *mDescriptor; + nsCacheAccessMode mAccessGranted; + nsresult mStatus; +}; + + +nsresult +nsCacheService::NotifyListener(nsCacheRequest * request, + nsICacheEntryDescriptor * descriptor, + nsCacheAccessMode accessGranted, + nsresult status) +{ + NS_ASSERTION(request->mThread, "no thread set in async request!"); + + // Swap ownership, and release listener on target thread... + nsICacheListener *listener = request->mListener; + request->mListener = nullptr; + + nsCOMPtr<nsIRunnable> ev = + new nsCacheListenerEvent(listener, descriptor, + accessGranted, status); + if (!ev) { + // Better to leak listener and descriptor if we fail because we don't + // want to destroy them inside the cache service lock or on potentially + // the wrong thread. + return NS_ERROR_OUT_OF_MEMORY; + } + + return request->mThread->Dispatch(ev, NS_DISPATCH_NORMAL); +} + + +nsresult +nsCacheService::ProcessRequest(nsCacheRequest * request, + bool calledFromOpenCacheEntry, + nsICacheEntryDescriptor ** result) +{ + // !!! must be called with mLock held !!! + nsresult rv; + nsCacheEntry * entry = nullptr; + nsCacheEntry * doomedEntry = nullptr; + nsCacheAccessMode accessGranted = nsICache::ACCESS_NONE; + if (result) *result = nullptr; + + while(1) { // Activate entry loop + rv = ActivateEntry(request, &entry, &doomedEntry); // get the entry for this request + if (NS_FAILED(rv)) break; + + while(1) { // Request Access loop + NS_ASSERTION(entry, "no entry in Request Access loop!"); + // entry->RequestAccess queues request on entry + rv = entry->RequestAccess(request, &accessGranted); + if (rv != NS_ERROR_CACHE_WAIT_FOR_VALIDATION) break; + + if (request->IsBlocking()) { + if (request->mListener) { + // async exits - validate, doom, or close will resume + return rv; + } + + // XXX this is probably wrong... + Unlock(); + rv = request->WaitForValidation(); + Lock(LOCK_TELEM(NSCACHESERVICE_PROCESSREQUEST)); + } + + PR_REMOVE_AND_INIT_LINK(request); + if (NS_FAILED(rv)) break; // non-blocking mode returns WAIT_FOR_VALIDATION error + // okay, we're ready to process this request, request access again + } + if (rv != NS_ERROR_CACHE_ENTRY_DOOMED) break; + + if (entry->IsNotInUse()) { + // this request was the last one keeping it around, so get rid of it + DeactivateEntry(entry); + } + // loop back around to look for another entry + } + + if (NS_SUCCEEDED(rv) && request->mProfileDir) { + // Custom cache directory has been demanded. Preset the cache device. + if (entry->StoragePolicy() != nsICache::STORE_OFFLINE) { + // Failsafe check: this is implemented only for offline cache atm. + rv = NS_ERROR_FAILURE; + } else { + RefPtr<nsOfflineCacheDevice> customCacheDevice; + rv = GetCustomOfflineDevice(request->mProfileDir, -1, + getter_AddRefs(customCacheDevice)); + if (NS_SUCCEEDED(rv)) + entry->SetCustomCacheDevice(customCacheDevice); + } + } + + nsICacheEntryDescriptor *descriptor = nullptr; + + if (NS_SUCCEEDED(rv)) + rv = entry->CreateDescriptor(request, accessGranted, &descriptor); + + // If doomedEntry is set, ActivatEntry() doomed an existing entry and + // created a new one for that cache-key. However, any pending requests + // on the doomed entry were not processed and we need to do that here. + // This must be done after adding the created entry to list of active + // entries (which is done in ActivateEntry()) otherwise the hashkeys crash + // (see bug ##561313). It is also important to do this after creating a + // descriptor for this request, or some other request may end up being + // executed first for the newly created entry. + // Finally, it is worth to emphasize that if doomedEntry is set, + // ActivateEntry() created a new entry for the request, which will be + // initialized by RequestAccess() and they both should have returned NS_OK. + if (doomedEntry) { + (void) ProcessPendingRequests(doomedEntry); + if (doomedEntry->IsNotInUse()) + DeactivateEntry(doomedEntry); + doomedEntry = nullptr; + } + + if (request->mListener) { // Asynchronous + + if (NS_FAILED(rv) && calledFromOpenCacheEntry && request->IsBlocking()) + return rv; // skip notifying listener, just return rv to caller + + // call listener to report error or descriptor + nsresult rv2 = NotifyListener(request, descriptor, accessGranted, rv); + if (NS_FAILED(rv2) && NS_SUCCEEDED(rv)) { + rv = rv2; // trigger delete request + } + } else { // Synchronous + *result = descriptor; + } + return rv; +} + + +nsresult +nsCacheService::OpenCacheEntry(nsCacheSession * session, + const nsACString & key, + nsCacheAccessMode accessRequested, + bool blockingMode, + nsICacheListener * listener, + nsICacheEntryDescriptor ** result) +{ + CACHE_LOG_DEBUG(("Opening entry for session %p, key %s, mode %d, blocking %d\n", + session, PromiseFlatCString(key).get(), accessRequested, + blockingMode)); + if (result) + *result = nullptr; + + if (!gService || !gService->mInitialized) + return NS_ERROR_NOT_INITIALIZED; + + nsCacheRequest * request = nullptr; + + nsresult rv = gService->CreateRequest(session, + key, + accessRequested, + blockingMode, + listener, + &request); + if (NS_FAILED(rv)) return rv; + + CACHE_LOG_DEBUG(("Created request %p\n", request)); + + // Process the request on the background thread if we are on the main thread + // and the the request is asynchronous + if (NS_IsMainThread() && listener && gService->mCacheIOThread) { + nsCOMPtr<nsIRunnable> ev = + new nsProcessRequestEvent(request); + rv = DispatchToCacheIOThread(ev); + + // delete request if we didn't post the event + if (NS_FAILED(rv)) + delete request; + } + else { + + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHESERVICE_OPENCACHEENTRY)); + rv = gService->ProcessRequest(request, true, result); + + // delete requests that have completed + if (!(listener && blockingMode && + (rv == NS_ERROR_CACHE_WAIT_FOR_VALIDATION))) + delete request; + } + + return rv; +} + + +nsresult +nsCacheService::ActivateEntry(nsCacheRequest * request, + nsCacheEntry ** result, + nsCacheEntry ** doomedEntry) +{ + CACHE_LOG_DEBUG(("Activate entry for request %p\n", request)); + if (!mInitialized || mClearingEntries) + return NS_ERROR_NOT_AVAILABLE; + + nsresult rv = NS_OK; + + NS_ASSERTION(request != nullptr, "ActivateEntry called with no request"); + if (result) *result = nullptr; + if (doomedEntry) *doomedEntry = nullptr; + if ((!request) || (!result) || (!doomedEntry)) + return NS_ERROR_NULL_POINTER; + + // check if the request can be satisfied + if (!mEnableMemoryDevice && !request->IsStreamBased()) + return NS_ERROR_FAILURE; + if (!IsStorageEnabledForPolicy_Locked(request->StoragePolicy())) + return NS_ERROR_FAILURE; + + // search active entries (including those not bound to device) + nsCacheEntry *entry = mActiveEntries.GetEntry(&(request->mKey)); + CACHE_LOG_DEBUG(("Active entry for request %p is %p\n", request, entry)); + + if (!entry) { + // search cache devices for entry + bool collision = false; + entry = SearchCacheDevices(&(request->mKey), request->StoragePolicy(), &collision); + CACHE_LOG_DEBUG(("Device search for request %p returned %p\n", + request, entry)); + // When there is a hashkey collision just refuse to cache it... + if (collision) return NS_ERROR_CACHE_IN_USE; + + if (entry) entry->MarkInitialized(); + } else { + NS_ASSERTION(entry->IsActive(), "Inactive entry found in mActiveEntries!"); + } + + if (entry) { + ++mCacheHits; + entry->Fetched(); + } else { + ++mCacheMisses; + } + + if (entry && + ((request->AccessRequested() == nsICache::ACCESS_WRITE) || + ((request->StoragePolicy() != nsICache::STORE_OFFLINE) && + (entry->mExpirationTime <= SecondsFromPRTime(PR_Now()) && + request->WillDoomEntriesIfExpired())))) + + { + // this is FORCE-WRITE request or the entry has expired + // we doom entry without processing pending requests, but store it in + // doomedEntry which causes pending requests to be processed below + rv = DoomEntry_Internal(entry, false); + *doomedEntry = entry; + if (NS_FAILED(rv)) { + // XXX what to do? Increment FailedDooms counter? + } + entry = nullptr; + } + + if (!entry) { + if (! (request->AccessRequested() & nsICache::ACCESS_WRITE)) { + // this is a READ-ONLY request + rv = NS_ERROR_CACHE_KEY_NOT_FOUND; + goto error; + } + + entry = new nsCacheEntry(request->mKey, + request->IsStreamBased(), + request->StoragePolicy()); + if (!entry) + return NS_ERROR_OUT_OF_MEMORY; + + if (request->IsPrivate()) + entry->MarkPrivate(); + + entry->Fetched(); + ++mTotalEntries; + + // XXX we could perform an early bind in some cases based on storage policy + } + + if (!entry->IsActive()) { + rv = mActiveEntries.AddEntry(entry); + if (NS_FAILED(rv)) goto error; + CACHE_LOG_DEBUG(("Added entry %p to mActiveEntries\n", entry)); + entry->MarkActive(); // mark entry active, because it's now in mActiveEntries + } + *result = entry; + return NS_OK; + + error: + *result = nullptr; + delete entry; + return rv; +} + + +nsCacheEntry * +nsCacheService::SearchCacheDevices(nsCString * key, nsCacheStoragePolicy policy, bool *collision) +{ + Telemetry::AutoTimer<Telemetry::CACHE_DEVICE_SEARCH_2> timer; + nsCacheEntry * entry = nullptr; + + CACHE_LOG_DEBUG(("mMemoryDevice: 0x%p\n", mMemoryDevice)); + + *collision = false; + if ((policy == nsICache::STORE_ANYWHERE) || (policy == nsICache::STORE_IN_MEMORY)) { + // If there is no memory device, then there is nothing to search... + if (mMemoryDevice) { + entry = mMemoryDevice->FindEntry(key, collision); + CACHE_LOG_DEBUG(("Searching mMemoryDevice for key %s found: 0x%p, " + "collision: %d\n", key->get(), entry, collision)); + } + } + + if (!entry && + ((policy == nsICache::STORE_ANYWHERE) || (policy == nsICache::STORE_ON_DISK))) { + + if (mEnableDiskDevice) { + if (!mDiskDevice) { + nsresult rv = CreateDiskDevice(); + if (NS_FAILED(rv)) + return nullptr; + } + + entry = mDiskDevice->FindEntry(key, collision); + } + } + + if (!entry && (policy == nsICache::STORE_OFFLINE || + (policy == nsICache::STORE_ANYWHERE && + gIOService->IsOffline()))) { + + if (mEnableOfflineDevice) { + if (!mOfflineDevice) { + nsresult rv = CreateOfflineDevice(); + if (NS_FAILED(rv)) + return nullptr; + } + + entry = mOfflineDevice->FindEntry(key, collision); + } + } + + return entry; +} + + +nsCacheDevice * +nsCacheService::EnsureEntryHasDevice(nsCacheEntry * entry) +{ + nsCacheDevice * device = entry->CacheDevice(); + // return device if found, possibly null if the entry is doomed i.e prevent + // doomed entries to bind to a device (see e.g. bugs #548406 and #596443) + if (device || entry->IsDoomed()) return device; + + int64_t predictedDataSize = entry->PredictedDataSize(); + if (entry->IsStreamData() && entry->IsAllowedOnDisk() && mEnableDiskDevice) { + // this is the default + if (!mDiskDevice) { + (void)CreateDiskDevice(); // ignore the error (check for mDiskDevice instead) + } + + if (mDiskDevice) { + // Bypass the cache if Content-Length says the entry will be too big + if (predictedDataSize != -1 && + mDiskDevice->EntryIsTooBig(predictedDataSize)) { + DebugOnly<nsresult> rv = nsCacheService::DoomEntry(entry); + NS_ASSERTION(NS_SUCCEEDED(rv),"DoomEntry() failed."); + return nullptr; + } + + entry->MarkBinding(); // enter state of binding + nsresult rv = mDiskDevice->BindEntry(entry); + entry->ClearBinding(); // exit state of binding + if (NS_SUCCEEDED(rv)) + device = mDiskDevice; + } + } + + // if we can't use mDiskDevice, try mMemoryDevice + if (!device && mEnableMemoryDevice && entry->IsAllowedInMemory()) { + if (!mMemoryDevice) { + (void)CreateMemoryDevice(); // ignore the error (check for mMemoryDevice instead) + } + if (mMemoryDevice) { + // Bypass the cache if Content-Length says entry will be too big + if (predictedDataSize != -1 && + mMemoryDevice->EntryIsTooBig(predictedDataSize)) { + DebugOnly<nsresult> rv = nsCacheService::DoomEntry(entry); + NS_ASSERTION(NS_SUCCEEDED(rv),"DoomEntry() failed."); + return nullptr; + } + + entry->MarkBinding(); // enter state of binding + nsresult rv = mMemoryDevice->BindEntry(entry); + entry->ClearBinding(); // exit state of binding + if (NS_SUCCEEDED(rv)) + device = mMemoryDevice; + } + } + + if (!device && entry->IsStreamData() && + entry->IsAllowedOffline() && mEnableOfflineDevice) { + if (!mOfflineDevice) { + (void)CreateOfflineDevice(); // ignore the error (check for mOfflineDevice instead) + } + + device = entry->CustomCacheDevice() + ? entry->CustomCacheDevice() + : mOfflineDevice; + + if (device) { + entry->MarkBinding(); + nsresult rv = device->BindEntry(entry); + entry->ClearBinding(); + if (NS_FAILED(rv)) + device = nullptr; + } + } + + if (device) + entry->SetCacheDevice(device); + return device; +} + +nsresult +nsCacheService::DoomEntry(nsCacheEntry * entry) +{ + return gService->DoomEntry_Internal(entry, true); +} + + +nsresult +nsCacheService::DoomEntry_Internal(nsCacheEntry * entry, + bool doProcessPendingRequests) +{ + if (entry->IsDoomed()) return NS_OK; + + CACHE_LOG_DEBUG(("Dooming entry %p\n", entry)); + nsresult rv = NS_OK; + entry->MarkDoomed(); + + NS_ASSERTION(!entry->IsBinding(), "Dooming entry while binding device."); + nsCacheDevice * device = entry->CacheDevice(); + if (device) device->DoomEntry(entry); + + if (entry->IsActive()) { + // remove from active entries + mActiveEntries.RemoveEntry(entry); + CACHE_LOG_DEBUG(("Removed entry %p from mActiveEntries\n", entry)); + entry->MarkInactive(); + } + + // put on doom list to wait for descriptors to close + NS_ASSERTION(PR_CLIST_IS_EMPTY(entry), "doomed entry still on device list"); + PR_APPEND_LINK(entry, &mDoomedEntries); + + // handle pending requests only if we're supposed to + if (doProcessPendingRequests) { + // tell pending requests to get on with their lives... + rv = ProcessPendingRequests(entry); + + // All requests have been removed, but there may still be open descriptors + if (entry->IsNotInUse()) { + DeactivateEntry(entry); // tell device to get rid of it + } + } + return rv; +} + + +void +nsCacheService::OnProfileShutdown() +{ + if (!gService || !gService->mInitialized) { + // The cache service has been shut down, but someone is still holding + // a reference to it. Ignore this call. + return; + } + + { + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHESERVICE_ONPROFILESHUTDOWN)); + gService->mClearingEntries = true; + gService->DoomActiveEntries(nullptr); + } + + gService->CloseAllStreams(); + + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHESERVICE_ONPROFILESHUTDOWN)); + gService->ClearDoomList(); + + // Make sure to wait for any pending cache-operations before + // proceeding with destructive actions (bug #620660) + (void) SyncWithCacheIOThread(); + + if (gService->mDiskDevice && gService->mEnableDiskDevice) { + gService->mDiskDevice->Shutdown(); + } + gService->mEnableDiskDevice = false; + + if (gService->mOfflineDevice && gService->mEnableOfflineDevice) { + gService->mOfflineDevice->Shutdown(); + } + for (auto iter = gService->mCustomOfflineDevices.Iter(); + !iter.Done(); iter.Next()) { + iter.Data()->Shutdown(); + iter.Remove(); + } + + gService->mEnableOfflineDevice = false; + + if (gService->mMemoryDevice) { + // clear memory cache + gService->mMemoryDevice->EvictEntries(nullptr); + } + + gService->mClearingEntries = false; +} + + +void +nsCacheService::OnProfileChanged() +{ + if (!gService) return; + + CACHE_LOG_DEBUG(("nsCacheService::OnProfileChanged")); + + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHESERVICE_ONPROFILECHANGED)); + + gService->mEnableDiskDevice = gService->mObserver->DiskCacheEnabled(); + gService->mEnableOfflineDevice = gService->mObserver->OfflineCacheEnabled(); + gService->mEnableMemoryDevice = gService->mObserver->MemoryCacheEnabled(); + + if (gService->mDiskDevice) { + gService->mDiskDevice->SetCacheParentDirectory(gService->mObserver->DiskCacheParentDirectory()); + gService->mDiskDevice->SetCapacity(gService->mObserver->DiskCacheCapacity()); + + // XXX initialization of mDiskDevice could be made lazily, if mEnableDiskDevice is false + nsresult rv = gService->mDiskDevice->Init(); + if (NS_FAILED(rv)) { + NS_ERROR("nsCacheService::OnProfileChanged: Re-initializing disk device failed"); + gService->mEnableDiskDevice = false; + // XXX delete mDiskDevice? + } + } + + if (gService->mOfflineDevice) { + gService->mOfflineDevice->SetCacheParentDirectory(gService->mObserver->OfflineCacheParentDirectory()); + gService->mOfflineDevice->SetCapacity(gService->mObserver->OfflineCacheCapacity()); + + // XXX initialization of mOfflineDevice could be made lazily, if mEnableOfflineDevice is false + nsresult rv = gService->mOfflineDevice->InitWithSqlite(gService->mStorageService); + if (NS_FAILED(rv)) { + NS_ERROR("nsCacheService::OnProfileChanged: Re-initializing offline device failed"); + gService->mEnableOfflineDevice = false; + // XXX delete mOfflineDevice? + } + } + + // If memoryDevice exists, reset its size to the new profile + if (gService->mMemoryDevice) { + if (gService->mEnableMemoryDevice) { + // make sure that capacity is reset to the right value + int32_t capacity = gService->mObserver->MemoryCacheCapacity(); + CACHE_LOG_DEBUG(("Resetting memory device capacity to %d\n", + capacity)); + gService->mMemoryDevice->SetCapacity(capacity); + } else { + // tell memory device to evict everything + CACHE_LOG_DEBUG(("memory device disabled\n")); + gService->mMemoryDevice->SetCapacity(0); + // Don't delete memory device, because some entries may be active still... + } + } +} + + +void +nsCacheService::SetDiskCacheEnabled(bool enabled) +{ + if (!gService) return; + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHESERVICE_SETDISKCACHEENABLED)); + gService->mEnableDiskDevice = enabled; +} + + +void +nsCacheService::SetDiskCacheCapacity(int32_t capacity) +{ + if (!gService) return; + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHESERVICE_SETDISKCACHECAPACITY)); + + if (gService->mDiskDevice) { + gService->mDiskDevice->SetCapacity(capacity); + } + + gService->mEnableDiskDevice = gService->mObserver->DiskCacheEnabled(); +} + +void +nsCacheService::SetDiskCacheMaxEntrySize(int32_t maxSize) +{ + if (!gService) return; + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHESERVICE_SETDISKCACHEMAXENTRYSIZE)); + + if (gService->mDiskDevice) { + gService->mDiskDevice->SetMaxEntrySize(maxSize); + } +} + +void +nsCacheService::SetMemoryCacheMaxEntrySize(int32_t maxSize) +{ + if (!gService) return; + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHESERVICE_SETMEMORYCACHEMAXENTRYSIZE)); + + if (gService->mMemoryDevice) { + gService->mMemoryDevice->SetMaxEntrySize(maxSize); + } +} + +void +nsCacheService::SetOfflineCacheEnabled(bool enabled) +{ + if (!gService) return; + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHESERVICE_SETOFFLINECACHEENABLED)); + gService->mEnableOfflineDevice = enabled; +} + +void +nsCacheService::SetOfflineCacheCapacity(int32_t capacity) +{ + if (!gService) return; + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHESERVICE_SETOFFLINECACHECAPACITY)); + + if (gService->mOfflineDevice) { + gService->mOfflineDevice->SetCapacity(capacity); + } + + gService->mEnableOfflineDevice = gService->mObserver->OfflineCacheEnabled(); +} + + +void +nsCacheService::SetMemoryCache() +{ + if (!gService) return; + + CACHE_LOG_DEBUG(("nsCacheService::SetMemoryCache")); + + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHESERVICE_SETMEMORYCACHE)); + + gService->mEnableMemoryDevice = gService->mObserver->MemoryCacheEnabled(); + + if (gService->mEnableMemoryDevice) { + if (gService->mMemoryDevice) { + int32_t capacity = gService->mObserver->MemoryCacheCapacity(); + // make sure that capacity is reset to the right value + CACHE_LOG_DEBUG(("Resetting memory device capacity to %d\n", + capacity)); + gService->mMemoryDevice->SetCapacity(capacity); + } + } else { + if (gService->mMemoryDevice) { + // tell memory device to evict everything + CACHE_LOG_DEBUG(("memory device disabled\n")); + gService->mMemoryDevice->SetCapacity(0); + // Don't delete memory device, because some entries may be active still... + } + } +} + + +/****************************************************************************** + * static methods for nsCacheEntryDescriptor + *****************************************************************************/ +void +nsCacheService::CloseDescriptor(nsCacheEntryDescriptor * descriptor) +{ + // ask entry to remove descriptor + nsCacheEntry * entry = descriptor->CacheEntry(); + bool doomEntry; + bool stillActive = entry->RemoveDescriptor(descriptor, &doomEntry); + + if (!entry->IsValid()) { + gService->ProcessPendingRequests(entry); + } + + if (doomEntry) { + gService->DoomEntry_Internal(entry, true); + return; + } + + if (!stillActive) { + gService->DeactivateEntry(entry); + } +} + + +nsresult +nsCacheService::GetFileForEntry(nsCacheEntry * entry, + nsIFile ** result) +{ + nsCacheDevice * device = gService->EnsureEntryHasDevice(entry); + if (!device) return NS_ERROR_UNEXPECTED; + + return device->GetFileForEntry(entry, result); +} + + +nsresult +nsCacheService::OpenInputStreamForEntry(nsCacheEntry * entry, + nsCacheAccessMode mode, + uint32_t offset, + nsIInputStream ** result) +{ + nsCacheDevice * device = gService->EnsureEntryHasDevice(entry); + if (!device) return NS_ERROR_UNEXPECTED; + + return device->OpenInputStreamForEntry(entry, mode, offset, result); +} + +nsresult +nsCacheService::OpenOutputStreamForEntry(nsCacheEntry * entry, + nsCacheAccessMode mode, + uint32_t offset, + nsIOutputStream ** result) +{ + nsCacheDevice * device = gService->EnsureEntryHasDevice(entry); + if (!device) return NS_ERROR_UNEXPECTED; + + return device->OpenOutputStreamForEntry(entry, mode, offset, result); +} + + +nsresult +nsCacheService::OnDataSizeChange(nsCacheEntry * entry, int32_t deltaSize) +{ + nsCacheDevice * device = gService->EnsureEntryHasDevice(entry); + if (!device) return NS_ERROR_UNEXPECTED; + + return device->OnDataSizeChange(entry, deltaSize); +} + +void +nsCacheService::LockAcquired() +{ + MutexAutoLock lock(mTimeStampLock); + mLockAcquiredTimeStamp = TimeStamp::Now(); +} + +void +nsCacheService::LockReleased() +{ + MutexAutoLock lock(mTimeStampLock); + mLockAcquiredTimeStamp = TimeStamp(); +} + +void +nsCacheService::Lock() +{ + gService->mLock.Lock(); + gService->LockAcquired(); +} + +void +nsCacheService::Lock(mozilla::Telemetry::ID mainThreadLockerID) +{ + mozilla::Telemetry::ID lockerID; + mozilla::Telemetry::ID generalID; + + if (NS_IsMainThread()) { + lockerID = mainThreadLockerID; + generalID = mozilla::Telemetry::CACHE_SERVICE_LOCK_WAIT_MAINTHREAD_2; + } else { + lockerID = mozilla::Telemetry::HistogramCount; + generalID = mozilla::Telemetry::CACHE_SERVICE_LOCK_WAIT_2; + } + + TimeStamp start(TimeStamp::Now()); + + nsCacheService::Lock(); + + TimeStamp stop(TimeStamp::Now()); + + // Telemetry isn't thread safe on its own, but this is OK because we're + // protecting it with the cache lock. + if (lockerID != mozilla::Telemetry::HistogramCount) { + mozilla::Telemetry::AccumulateTimeDelta(lockerID, start, stop); + } + mozilla::Telemetry::AccumulateTimeDelta(generalID, start, stop); +} + +void +nsCacheService::Unlock() +{ + gService->mLock.AssertCurrentThreadOwns(); + + nsTArray<nsISupports*> doomed; + doomed.SwapElements(gService->mDoomedObjects); + + gService->LockReleased(); + gService->mLock.Unlock(); + + for (uint32_t i = 0; i < doomed.Length(); ++i) + doomed[i]->Release(); +} + +void +nsCacheService::ReleaseObject_Locked(nsISupports * obj, + nsIEventTarget * target) +{ + gService->mLock.AssertCurrentThreadOwns(); + + bool isCur; + if (!target || (NS_SUCCEEDED(target->IsOnCurrentThread(&isCur)) && isCur)) { + gService->mDoomedObjects.AppendElement(obj); + } else { + NS_ProxyRelease(target, dont_AddRef(obj)); + } +} + + +nsresult +nsCacheService::SetCacheElement(nsCacheEntry * entry, nsISupports * element) +{ + entry->SetData(element); + entry->TouchData(); + return NS_OK; +} + + +nsresult +nsCacheService::ValidateEntry(nsCacheEntry * entry) +{ + nsCacheDevice * device = gService->EnsureEntryHasDevice(entry); + if (!device) return NS_ERROR_UNEXPECTED; + + entry->MarkValid(); + nsresult rv = gService->ProcessPendingRequests(entry); + NS_ASSERTION(rv == NS_OK, "ProcessPendingRequests failed."); + // XXX what else should be done? + + return rv; +} + + +int32_t +nsCacheService::CacheCompressionLevel() +{ + int32_t level = gService->mObserver->CacheCompressionLevel(); + return level; +} + + +void +nsCacheService::DeactivateEntry(nsCacheEntry * entry) +{ + CACHE_LOG_DEBUG(("Deactivating entry %p\n", entry)); + nsresult rv = NS_OK; + NS_ASSERTION(entry->IsNotInUse(), "### deactivating an entry while in use!"); + nsCacheDevice * device = nullptr; + + if (mMaxDataSize < entry->DataSize() ) mMaxDataSize = entry->DataSize(); + if (mMaxMetaSize < entry->MetaDataSize() ) mMaxMetaSize = entry->MetaDataSize(); + + if (entry->IsDoomed()) { + // remove from Doomed list + PR_REMOVE_AND_INIT_LINK(entry); + } else if (entry->IsActive()) { + // remove from active entries + mActiveEntries.RemoveEntry(entry); + CACHE_LOG_DEBUG(("Removed deactivated entry %p from mActiveEntries\n", + entry)); + entry->MarkInactive(); + + // bind entry if necessary to store meta-data + device = EnsureEntryHasDevice(entry); + if (!device) { + CACHE_LOG_DEBUG(("DeactivateEntry: unable to bind active " + "entry %p\n", + entry)); + NS_WARNING("DeactivateEntry: unable to bind active entry\n"); + return; + } + } else { + // if mInitialized == false, + // then we're shutting down and this state is okay. + NS_ASSERTION(!mInitialized, "DeactivateEntry: bad cache entry state."); + } + + device = entry->CacheDevice(); + if (device) { + rv = device->DeactivateEntry(entry); + if (NS_FAILED(rv)) { + // increment deactivate failure count + ++mDeactivateFailures; + } + } else { + // increment deactivating unbound entry statistic + ++mDeactivatedUnboundEntries; + delete entry; // because no one else will + } +} + + +nsresult +nsCacheService::ProcessPendingRequests(nsCacheEntry * entry) +{ + nsresult rv = NS_OK; + nsCacheRequest * request = (nsCacheRequest *)PR_LIST_HEAD(&entry->mRequestQ); + nsCacheRequest * nextRequest; + bool newWriter = false; + + CACHE_LOG_DEBUG(("ProcessPendingRequests for %sinitialized %s %salid entry %p\n", + (entry->IsInitialized()?"" : "Un"), + (entry->IsDoomed()?"DOOMED" : ""), + (entry->IsValid()? "V":"Inv"), entry)); + + if (request == &entry->mRequestQ) return NS_OK; // no queued requests + + if (!entry->IsDoomed() && entry->IsInvalid()) { + // 1st descriptor closed w/o MarkValid() + NS_ASSERTION(PR_CLIST_IS_EMPTY(&entry->mDescriptorQ), "shouldn't be here with open descriptors"); + +#if DEBUG + // verify no ACCESS_WRITE requests(shouldn't have any of these) + while (request != &entry->mRequestQ) { + NS_ASSERTION(request->AccessRequested() != nsICache::ACCESS_WRITE, + "ACCESS_WRITE request should have been given a new entry"); + request = (nsCacheRequest *)PR_NEXT_LINK(request); + } + request = (nsCacheRequest *)PR_LIST_HEAD(&entry->mRequestQ); +#endif + // find first request with ACCESS_READ_WRITE (if any) and promote it to 1st writer + while (request != &entry->mRequestQ) { + if (request->AccessRequested() == nsICache::ACCESS_READ_WRITE) { + newWriter = true; + CACHE_LOG_DEBUG((" promoting request %p to 1st writer\n", request)); + break; + } + + request = (nsCacheRequest *)PR_NEXT_LINK(request); + } + + if (request == &entry->mRequestQ) // no requests asked for ACCESS_READ_WRITE, back to top + request = (nsCacheRequest *)PR_LIST_HEAD(&entry->mRequestQ); + + // XXX what should we do if there are only READ requests in queue? + // XXX serialize their accesses, give them only read access, but force them to check validate flag? + // XXX or do readers simply presume the entry is valid + // See fix for bug #467392 below + } + + nsCacheAccessMode accessGranted = nsICache::ACCESS_NONE; + + while (request != &entry->mRequestQ) { + nextRequest = (nsCacheRequest *)PR_NEXT_LINK(request); + CACHE_LOG_DEBUG((" %sync request %p for %p\n", + (request->mListener?"As":"S"), request, entry)); + + if (request->mListener) { + + // Async request + PR_REMOVE_AND_INIT_LINK(request); + + if (entry->IsDoomed()) { + rv = ProcessRequest(request, false, nullptr); + if (rv == NS_ERROR_CACHE_WAIT_FOR_VALIDATION) + rv = NS_OK; + else + delete request; + + if (NS_FAILED(rv)) { + // XXX what to do? + } + } else if (entry->IsValid() || newWriter) { + rv = entry->RequestAccess(request, &accessGranted); + NS_ASSERTION(NS_SUCCEEDED(rv), + "if entry is valid, RequestAccess must succeed."); + // XXX if (newWriter) NS_ASSERTION( accessGranted == request->AccessRequested(), "why not?"); + + // entry->CreateDescriptor dequeues request, and queues descriptor + nsICacheEntryDescriptor *descriptor = nullptr; + rv = entry->CreateDescriptor(request, + accessGranted, + &descriptor); + + // post call to listener to report error or descriptor + rv = NotifyListener(request, descriptor, accessGranted, rv); + delete request; + if (NS_FAILED(rv)) { + // XXX what to do? + } + + } else { + // read-only request to an invalid entry - need to wait for + // the entry to become valid so we post an event to process + // the request again later (bug #467392) + nsCOMPtr<nsIRunnable> ev = + new nsProcessRequestEvent(request); + rv = DispatchToCacheIOThread(ev); + if (NS_FAILED(rv)) { + delete request; // avoid leak + } + } + } else { + + // Synchronous request + request->WakeUp(); + } + if (newWriter) break; // process remaining requests after validation + request = nextRequest; + } + + return NS_OK; +} + +bool +nsCacheService::IsDoomListEmpty() +{ + nsCacheEntry * entry = (nsCacheEntry *)PR_LIST_HEAD(&mDoomedEntries); + return &mDoomedEntries == entry; +} + +void +nsCacheService::ClearDoomList() +{ + nsCacheEntry * entry = (nsCacheEntry *)PR_LIST_HEAD(&mDoomedEntries); + + while (entry != &mDoomedEntries) { + nsCacheEntry * next = (nsCacheEntry *)PR_NEXT_LINK(entry); + + entry->DetachDescriptors(); + DeactivateEntry(entry); + entry = next; + } +} + +void +nsCacheService::DoomActiveEntries(DoomCheckFn check) +{ + AutoTArray<nsCacheEntry*, 8> array; + + for (auto iter = mActiveEntries.Iter(); !iter.Done(); iter.Next()) { + nsCacheEntry* entry = + static_cast<nsCacheEntryHashTableEntry*>(iter.Get())->cacheEntry; + + if (check && !check(entry)) { + continue; + } + + array.AppendElement(entry); + + // entry is being removed from the active entry list + entry->MarkInactive(); + iter.Remove(); + } + + uint32_t count = array.Length(); + for (uint32_t i = 0; i < count; ++i) { + DoomEntry_Internal(array[i], true); + } +} + +void +nsCacheService::CloseAllStreams() +{ + nsTArray<RefPtr<nsCacheEntryDescriptor::nsInputStreamWrapper> > inputs; + nsTArray<RefPtr<nsCacheEntryDescriptor::nsOutputStreamWrapper> > outputs; + + { + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHESERVICE_CLOSEALLSTREAMS)); + + nsTArray<nsCacheEntry*> entries; + +#if DEBUG + // make sure there is no active entry + for (auto iter = mActiveEntries.Iter(); !iter.Done(); iter.Next()) { + auto entry = static_cast<nsCacheEntryHashTableEntry*>(iter.Get()); + entries.AppendElement(entry->cacheEntry); + } + NS_ASSERTION(entries.IsEmpty(), "Bad state"); +#endif + + // Get doomed entries + nsCacheEntry * entry = (nsCacheEntry *)PR_LIST_HEAD(&mDoomedEntries); + while (entry != &mDoomedEntries) { + nsCacheEntry * next = (nsCacheEntry *)PR_NEXT_LINK(entry); + entries.AppendElement(entry); + entry = next; + } + + // Iterate through all entries and collect input and output streams + for (size_t i = 0; i < entries.Length(); i++) { + entry = entries.ElementAt(i); + + nsTArray<RefPtr<nsCacheEntryDescriptor> > descs; + entry->GetDescriptors(descs); + + for (uint32_t j = 0 ; j < descs.Length() ; j++) { + if (descs[j]->mOutputWrapper) + outputs.AppendElement(descs[j]->mOutputWrapper); + + for (size_t k = 0; k < descs[j]->mInputWrappers.Length(); k++) + inputs.AppendElement(descs[j]->mInputWrappers[k]); + } + } + } + + uint32_t i; + for (i = 0 ; i < inputs.Length() ; i++) + inputs[i]->Close(); + + for (i = 0 ; i < outputs.Length() ; i++) + outputs[i]->Close(); +} + + +bool +nsCacheService::GetClearingEntries() +{ + AssertOwnsLock(); + return gService->mClearingEntries; +} + +// static +void nsCacheService::GetCacheBaseDirectoty(nsIFile ** result) +{ + *result = nullptr; + if (!gService || !gService->mObserver) + return; + + nsCOMPtr<nsIFile> directory = + gService->mObserver->DiskCacheParentDirectory(); + if (!directory) + return; + + directory->Clone(result); +} + +// static +void nsCacheService::GetDiskCacheDirectory(nsIFile ** result) +{ + nsCOMPtr<nsIFile> directory; + GetCacheBaseDirectoty(getter_AddRefs(directory)); + if (!directory) + return; + + nsresult rv = directory->AppendNative(NS_LITERAL_CSTRING("Cache")); + if (NS_FAILED(rv)) + return; + + directory.forget(result); +} + +// static +void nsCacheService::GetAppCacheDirectory(nsIFile ** result) +{ + nsCOMPtr<nsIFile> directory; + GetCacheBaseDirectoty(getter_AddRefs(directory)); + if (!directory) + return; + + nsresult rv = directory->AppendNative(NS_LITERAL_CSTRING("OfflineCache")); + if (NS_FAILED(rv)) + return; + + directory.forget(result); +} + + +void +nsCacheService::LogCacheStatistics() +{ + uint32_t hitPercentage = (uint32_t)((((double)mCacheHits) / + ((double)(mCacheHits + mCacheMisses))) * 100); + CACHE_LOG_INFO(("\nCache Service Statistics:\n\n")); + CACHE_LOG_INFO((" TotalEntries = %d\n", mTotalEntries)); + CACHE_LOG_INFO((" Cache Hits = %d\n", mCacheHits)); + CACHE_LOG_INFO((" Cache Misses = %d\n", mCacheMisses)); + CACHE_LOG_INFO((" Cache Hit %% = %d%%\n", hitPercentage)); + CACHE_LOG_INFO((" Max Key Length = %d\n", mMaxKeyLength)); + CACHE_LOG_INFO((" Max Meta Size = %d\n", mMaxMetaSize)); + CACHE_LOG_INFO((" Max Data Size = %d\n", mMaxDataSize)); + CACHE_LOG_INFO(("\n")); + CACHE_LOG_INFO((" Deactivate Failures = %d\n", + mDeactivateFailures)); + CACHE_LOG_INFO((" Deactivated Unbound Entries = %d\n", + mDeactivatedUnboundEntries)); +} + +nsresult +nsCacheService::SetDiskSmartSize() +{ + nsCacheServiceAutoLock lock(LOCK_TELEM(NSCACHESERVICE_SETDISKSMARTSIZE)); + + if (!gService) return NS_ERROR_NOT_AVAILABLE; + + return gService->SetDiskSmartSize_Locked(); +} + +nsresult +nsCacheService::SetDiskSmartSize_Locked() +{ + nsresult rv; + + if (mozilla::net::CacheObserver::UseNewCache()) { + return NS_ERROR_NOT_AVAILABLE; + } + + if (!mObserver->DiskCacheParentDirectory()) + return NS_ERROR_NOT_AVAILABLE; + + if (!mDiskDevice) + return NS_ERROR_NOT_AVAILABLE; + + if (!mObserver->SmartSizeEnabled()) + return NS_ERROR_NOT_AVAILABLE; + + nsAutoString cachePath; + rv = mObserver->DiskCacheParentDirectory()->GetPath(cachePath); + if (NS_SUCCEEDED(rv)) { + nsCOMPtr<nsIRunnable> event = + new nsGetSmartSizeEvent(cachePath, mDiskDevice->getCacheSize(), + mObserver->ShouldUseOldMaxSmartSize()); + DispatchToCacheIOThread(event); + } else { + return NS_ERROR_FAILURE; + } + + return NS_OK; +} + +void +nsCacheService::MoveOrRemoveDiskCache(nsIFile *aOldCacheDir, + nsIFile *aNewCacheDir, + const char *aCacheSubdir) +{ + bool same; + if (NS_FAILED(aOldCacheDir->Equals(aNewCacheDir, &same)) || same) + return; + + nsCOMPtr<nsIFile> aOldCacheSubdir; + aOldCacheDir->Clone(getter_AddRefs(aOldCacheSubdir)); + + nsresult rv = aOldCacheSubdir->AppendNative( + nsDependentCString(aCacheSubdir)); + if (NS_FAILED(rv)) + return; + + bool exists; + if (NS_FAILED(aOldCacheSubdir->Exists(&exists)) || !exists) + return; + + nsCOMPtr<nsIFile> aNewCacheSubdir; + aNewCacheDir->Clone(getter_AddRefs(aNewCacheSubdir)); + + rv = aNewCacheSubdir->AppendNative(nsDependentCString(aCacheSubdir)); + if (NS_FAILED(rv)) + return; + + nsAutoCString newPath; + rv = aNewCacheSubdir->GetNativePath(newPath); + if (NS_FAILED(rv)) + return; + + if (NS_SUCCEEDED(aNewCacheSubdir->Exists(&exists)) && !exists) { + // New cache directory does not exist, try to move the old one here + // rename needs an empty target directory + + // Make sure the parent of the target sub-dir exists + rv = aNewCacheDir->Create(nsIFile::DIRECTORY_TYPE, 0777); + if (NS_SUCCEEDED(rv) || NS_ERROR_FILE_ALREADY_EXISTS == rv) { + nsAutoCString oldPath; + rv = aOldCacheSubdir->GetNativePath(oldPath); + if (NS_FAILED(rv)) + return; + if (rename(oldPath.get(), newPath.get()) == 0) + return; + } + } + + // Delay delete by 1 minute to avoid IO thrash on startup. + nsDeleteDir::DeleteDir(aOldCacheSubdir, false, 60000); +} + +static bool +IsEntryPrivate(nsCacheEntry* entry) +{ + return entry->IsPrivate(); +} + +void +nsCacheService::LeavePrivateBrowsing() +{ + nsCacheServiceAutoLock lock; + + gService->DoomActiveEntries(IsEntryPrivate); + + if (gService->mMemoryDevice) { + // clear memory cache + gService->mMemoryDevice->EvictPrivateEntries(); + } +} + +MOZ_DEFINE_MALLOC_SIZE_OF(DiskCacheDeviceMallocSizeOf) + +NS_IMETHODIMP +nsCacheService::CollectReports(nsIHandleReportCallback* aHandleReport, + nsISupports* aData, bool aAnonymize) +{ + size_t disk = 0; + if (mDiskDevice) { + nsCacheServiceAutoLock + lock(LOCK_TELEM(NSCACHESERVICE_DISKDEVICEHEAPSIZE)); + disk = mDiskDevice->SizeOfIncludingThis(DiskCacheDeviceMallocSizeOf); + } + + size_t memory = mMemoryDevice ? mMemoryDevice->TotalSize() : 0; + + MOZ_COLLECT_REPORT( + "explicit/network/disk-cache", KIND_HEAP, UNITS_BYTES, disk, + "Memory used by the network disk cache."); + + MOZ_COLLECT_REPORT( + "explicit/network/memory-cache", KIND_HEAP, UNITS_BYTES, memory, + "Memory used by the network memory cache."); + + return NS_OK; +} diff --git a/netwerk/cache/nsCacheService.h b/netwerk/cache/nsCacheService.h new file mode 100644 index 000000000..1751d4875 --- /dev/null +++ b/netwerk/cache/nsCacheService.h @@ -0,0 +1,392 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* vim: set ts=8 sts=4 et sw=4 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef _nsCacheService_h_ +#define _nsCacheService_h_ + +#include "nsICacheService.h" +#include "nsCacheSession.h" +#include "nsCacheDevice.h" +#include "nsCacheEntry.h" +#include "nsThreadUtils.h" +#include "nsICacheListener.h" +#include "nsIMemoryReporter.h" + +#include "prthread.h" +#include "nsIObserver.h" +#include "nsString.h" +#include "nsTArray.h" +#include "nsRefPtrHashtable.h" +#include "mozilla/CondVar.h" +#include "mozilla/Mutex.h" +#include "mozilla/Telemetry.h" + +class nsCacheRequest; +class nsCacheProfilePrefObserver; +class nsDiskCacheDevice; +class nsMemoryCacheDevice; +class nsOfflineCacheDevice; +class nsCacheServiceAutoLock; +class nsITimer; +class mozIStorageService; + + +/****************************************************************************** + * nsNotifyDoomListener + *****************************************************************************/ + +class nsNotifyDoomListener : public mozilla::Runnable { +public: + nsNotifyDoomListener(nsICacheListener *listener, + nsresult status) + : mListener(listener) // transfers reference + , mStatus(status) + {} + + NS_IMETHOD Run() override + { + mListener->OnCacheEntryDoomed(mStatus); + NS_RELEASE(mListener); + return NS_OK; + } + +private: + nsICacheListener *mListener; + nsresult mStatus; +}; + +/****************************************************************************** + * nsCacheService + ******************************************************************************/ + +class nsCacheService final : public nsICacheServiceInternal, + public nsIMemoryReporter +{ + virtual ~nsCacheService(); + +public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSICACHESERVICE + NS_DECL_NSICACHESERVICEINTERNAL + NS_DECL_NSIMEMORYREPORTER + + nsCacheService(); + + // Define a Create method to be used with a factory: + static nsresult + Create(nsISupports* outer, const nsIID& iid, void* *result); + + + /** + * Methods called by nsCacheSession + */ + static nsresult OpenCacheEntry(nsCacheSession * session, + const nsACString & key, + nsCacheAccessMode accessRequested, + bool blockingMode, + nsICacheListener * listener, + nsICacheEntryDescriptor ** result); + + static nsresult EvictEntriesForSession(nsCacheSession * session); + + static nsresult IsStorageEnabledForPolicy(nsCacheStoragePolicy storagePolicy, + bool * result); + + static nsresult DoomEntry(nsCacheSession *session, + const nsACString &key, + nsICacheListener *listener); + + /** + * Methods called by nsCacheEntryDescriptor + */ + + static void CloseDescriptor(nsCacheEntryDescriptor * descriptor); + + static nsresult GetFileForEntry(nsCacheEntry * entry, + nsIFile ** result); + + static nsresult OpenInputStreamForEntry(nsCacheEntry * entry, + nsCacheAccessMode mode, + uint32_t offset, + nsIInputStream ** result); + + static nsresult OpenOutputStreamForEntry(nsCacheEntry * entry, + nsCacheAccessMode mode, + uint32_t offset, + nsIOutputStream ** result); + + static nsresult OnDataSizeChange(nsCacheEntry * entry, int32_t deltaSize); + + static nsresult SetCacheElement(nsCacheEntry * entry, nsISupports * element); + + static nsresult ValidateEntry(nsCacheEntry * entry); + + static int32_t CacheCompressionLevel(); + + static bool GetClearingEntries(); + + static void GetCacheBaseDirectoty(nsIFile ** result); + static void GetDiskCacheDirectory(nsIFile ** result); + static void GetAppCacheDirectory(nsIFile ** result); + + /** + * Methods called by any cache classes + */ + + static + nsCacheService * GlobalInstance() { return gService; } + + static nsresult DoomEntry(nsCacheEntry * entry); + + static bool IsStorageEnabledForPolicy_Locked(nsCacheStoragePolicy policy); + + /** + * Called by disk cache to notify us to use the new max smart size + */ + static void MarkStartingFresh(); + + /** + * Methods called by nsApplicationCacheService + */ + + nsresult GetOfflineDevice(nsOfflineCacheDevice ** aDevice); + + /** + * Creates an offline cache device that works over a specific profile directory. + * A tool to preload offline cache for profiles different from the current + * application's profile directory. + */ + nsresult GetCustomOfflineDevice(nsIFile *aProfileDir, + int32_t aQuota, + nsOfflineCacheDevice **aDevice); + + // This method may be called to release an object while the cache service + // lock is being held. If a non-null target is specified and the target + // does not correspond to the current thread, then the release will be + // proxied to the specified target. Otherwise, the object will be added to + // the list of objects to be released when the cache service is unlocked. + static void ReleaseObject_Locked(nsISupports * object, + nsIEventTarget * target = nullptr); + + static nsresult DispatchToCacheIOThread(nsIRunnable* event); + + // Calling this method will block the calling thread until all pending + // events on the cache-io thread has finished. The calling thread must + // hold the cache-lock + static nsresult SyncWithCacheIOThread(); + + + /** + * Methods called by nsCacheProfilePrefObserver + */ + static void OnProfileShutdown(); + static void OnProfileChanged(); + + static void SetDiskCacheEnabled(bool enabled); + // Sets the disk cache capacity (in kilobytes) + static void SetDiskCacheCapacity(int32_t capacity); + // Set max size for a disk-cache entry (in KB). -1 disables limit up to + // 1/8th of disk cache size + static void SetDiskCacheMaxEntrySize(int32_t maxSize); + // Set max size for a memory-cache entry (in kilobytes). -1 disables + // limit up to 90% of memory cache size + static void SetMemoryCacheMaxEntrySize(int32_t maxSize); + + static void SetOfflineCacheEnabled(bool enabled); + // Sets the offline cache capacity (in kilobytes) + static void SetOfflineCacheCapacity(int32_t capacity); + + static void SetMemoryCache(); + + static void SetCacheCompressionLevel(int32_t level); + + // Starts smart cache size computation if disk device is available + static nsresult SetDiskSmartSize(); + + static void MoveOrRemoveDiskCache(nsIFile *aOldCacheDir, + nsIFile *aNewCacheDir, + const char *aCacheSubdir); + + nsresult Init(); + void Shutdown(); + + static bool IsInitialized() + { + if (!gService) { + return false; + } + return gService->mInitialized; + } + + static void AssertOwnsLock() + { gService->mLock.AssertCurrentThreadOwns(); } + + static void LeavePrivateBrowsing(); + bool IsDoomListEmpty(); + + typedef bool (*DoomCheckFn)(nsCacheEntry* entry); + + // Accessors to the disabled functionality + nsresult CreateSessionInternal(const char * clientID, + nsCacheStoragePolicy storagePolicy, + bool streamBased, + nsICacheSession **result); + nsresult VisitEntriesInternal(nsICacheVisitor *visitor); + nsresult EvictEntriesInternal(nsCacheStoragePolicy storagePolicy); + +private: + friend class nsCacheServiceAutoLock; + friend class nsOfflineCacheDevice; + friend class nsProcessRequestEvent; + friend class nsSetSmartSizeEvent; + friend class nsBlockOnCacheThreadEvent; + friend class nsSetDiskSmartSizeCallback; + friend class nsDoomEvent; + friend class nsDisableOldMaxSmartSizePrefEvent; + friend class nsDiskCacheMap; + friend class nsAsyncDoomEvent; + friend class nsCacheEntryDescriptor; + + /** + * Internal Methods + */ + + static void Lock(); + static void Lock(::mozilla::Telemetry::ID mainThreadLockerID); + static void Unlock(); + void LockAcquired(); + void LockReleased(); + + nsresult CreateDiskDevice(); + nsresult CreateOfflineDevice(); + nsresult CreateCustomOfflineDevice(nsIFile *aProfileDir, + int32_t aQuota, + nsOfflineCacheDevice **aDevice); + nsresult CreateMemoryDevice(); + + nsresult RemoveCustomOfflineDevice(nsOfflineCacheDevice *aDevice); + + nsresult CreateRequest(nsCacheSession * session, + const nsACString & clientKey, + nsCacheAccessMode accessRequested, + bool blockingMode, + nsICacheListener * listener, + nsCacheRequest ** request); + + nsresult DoomEntry_Internal(nsCacheEntry * entry, + bool doProcessPendingRequests); + + nsresult EvictEntriesForClient(const char * clientID, + nsCacheStoragePolicy storagePolicy); + + // Notifies request listener asynchronously on the request's thread, and + // releases the descriptor on the request's thread. If this method fails, + // the descriptor is not released. + nsresult NotifyListener(nsCacheRequest * request, + nsICacheEntryDescriptor * descriptor, + nsCacheAccessMode accessGranted, + nsresult error); + + nsresult ActivateEntry(nsCacheRequest * request, + nsCacheEntry ** entry, + nsCacheEntry ** doomedEntry); + + nsCacheDevice * EnsureEntryHasDevice(nsCacheEntry * entry); + + nsCacheEntry * SearchCacheDevices(nsCString * key, nsCacheStoragePolicy policy, bool *collision); + + void DeactivateEntry(nsCacheEntry * entry); + + nsresult ProcessRequest(nsCacheRequest * request, + bool calledFromOpenCacheEntry, + nsICacheEntryDescriptor ** result); + + nsresult ProcessPendingRequests(nsCacheEntry * entry); + + void ClearDoomList(void); + void DoomActiveEntries(DoomCheckFn check); + void CloseAllStreams(); + void FireClearNetworkCacheStoredAnywhereNotification(); + + void LogCacheStatistics(); + + nsresult SetDiskSmartSize_Locked(); + + /** + * Data Members + */ + + static nsCacheService * gService; // there can be only one... + + nsCOMPtr<mozIStorageService> mStorageService; + + nsCacheProfilePrefObserver * mObserver; + + mozilla::Mutex mLock; + mozilla::CondVar mCondVar; + bool mNotified; + + mozilla::Mutex mTimeStampLock; + mozilla::TimeStamp mLockAcquiredTimeStamp; + + nsCOMPtr<nsIThread> mCacheIOThread; + + nsTArray<nsISupports*> mDoomedObjects; + nsCOMPtr<nsITimer> mSmartSizeTimer; + + bool mInitialized; + bool mClearingEntries; + + bool mEnableMemoryDevice; + bool mEnableDiskDevice; + bool mEnableOfflineDevice; + + nsMemoryCacheDevice * mMemoryDevice; + nsDiskCacheDevice * mDiskDevice; + nsOfflineCacheDevice * mOfflineDevice; + + nsRefPtrHashtable<nsStringHashKey, nsOfflineCacheDevice> mCustomOfflineDevices; + + nsCacheEntryHashTable mActiveEntries; + PRCList mDoomedEntries; + + // stats + + uint32_t mTotalEntries; + uint32_t mCacheHits; + uint32_t mCacheMisses; + uint32_t mMaxKeyLength; + uint32_t mMaxDataSize; + uint32_t mMaxMetaSize; + + // Unexpected error totals + uint32_t mDeactivateFailures; + uint32_t mDeactivatedUnboundEntries; +}; + +/****************************************************************************** + * nsCacheServiceAutoLock + ******************************************************************************/ + +#define LOCK_TELEM(x) \ + (::mozilla::Telemetry::CACHE_SERVICE_LOCK_WAIT_MAINTHREAD_##x) + +// Instantiate this class to acquire the cache service lock for a particular +// execution scope. +class nsCacheServiceAutoLock { +public: + nsCacheServiceAutoLock() { + nsCacheService::Lock(); + } + explicit nsCacheServiceAutoLock(mozilla::Telemetry::ID mainThreadLockerID) { + nsCacheService::Lock(mainThreadLockerID); + } + ~nsCacheServiceAutoLock() { + nsCacheService::Unlock(); + } +}; + +#endif // _nsCacheService_h_ diff --git a/netwerk/cache/nsCacheSession.cpp b/netwerk/cache/nsCacheSession.cpp new file mode 100644 index 000000000..d82203aec --- /dev/null +++ b/netwerk/cache/nsCacheSession.cpp @@ -0,0 +1,147 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsCacheSession.h" +#include "nsCacheService.h" +#include "nsCRT.h" +#include "nsThreadUtils.h" + +NS_IMPL_ISUPPORTS(nsCacheSession, nsICacheSession) + +nsCacheSession::nsCacheSession(const char * clientID, + nsCacheStoragePolicy storagePolicy, + bool streamBased) + : mClientID(clientID), + mInfo(0) +{ + SetStoragePolicy(storagePolicy); + + if (streamBased) MarkStreamBased(); + else SetStoragePolicy(nsICache::STORE_IN_MEMORY); + + MarkPublic(); + + MarkDoomEntriesIfExpired(); +} + +nsCacheSession::~nsCacheSession() +{ + /* destructor code */ + // notify service we are going away? +} + + +NS_IMETHODIMP nsCacheSession::GetDoomEntriesIfExpired(bool *result) +{ + NS_ENSURE_ARG_POINTER(result); + *result = WillDoomEntriesIfExpired(); + return NS_OK; +} + + +NS_IMETHODIMP nsCacheSession::SetProfileDirectory(nsIFile *profileDir) +{ + if (StoragePolicy() != nsICache::STORE_OFFLINE && profileDir) { + // Profile directory override is currently implemented only for + // offline cache. This is an early failure to prevent the request + // being processed before it would fail later because of inability + // to assign a cache base dir. + return NS_ERROR_UNEXPECTED; + } + + mProfileDir = profileDir; + return NS_OK; +} + +NS_IMETHODIMP nsCacheSession::GetProfileDirectory(nsIFile **profileDir) +{ + if (mProfileDir) + NS_ADDREF(*profileDir = mProfileDir); + else + *profileDir = nullptr; + + return NS_OK; +} + + +NS_IMETHODIMP nsCacheSession::SetDoomEntriesIfExpired(bool doomEntriesIfExpired) +{ + if (doomEntriesIfExpired) MarkDoomEntriesIfExpired(); + else ClearDoomEntriesIfExpired(); + return NS_OK; +} + + +NS_IMETHODIMP +nsCacheSession::OpenCacheEntry(const nsACString & key, + nsCacheAccessMode accessRequested, + bool blockingMode, + nsICacheEntryDescriptor ** result) +{ + nsresult rv; + + if (NS_IsMainThread()) + rv = NS_ERROR_NOT_AVAILABLE; + else + rv = nsCacheService::OpenCacheEntry(this, + key, + accessRequested, + blockingMode, + nullptr, // no listener + result); + return rv; +} + + +NS_IMETHODIMP nsCacheSession::AsyncOpenCacheEntry(const nsACString & key, + nsCacheAccessMode accessRequested, + nsICacheListener *listener, + bool noWait) +{ + nsresult rv; + rv = nsCacheService::OpenCacheEntry(this, + key, + accessRequested, + !noWait, + listener, + nullptr); // no result + + if (rv == NS_ERROR_CACHE_WAIT_FOR_VALIDATION) rv = NS_OK; + return rv; +} + +NS_IMETHODIMP nsCacheSession::EvictEntries() +{ + return nsCacheService::EvictEntriesForSession(this); +} + + +NS_IMETHODIMP nsCacheSession::IsStorageEnabled(bool *result) +{ + + return nsCacheService::IsStorageEnabledForPolicy(StoragePolicy(), result); +} + +NS_IMETHODIMP nsCacheSession::DoomEntry(const nsACString &key, + nsICacheListener *listener) +{ + return nsCacheService::DoomEntry(this, key, listener); +} + +NS_IMETHODIMP nsCacheSession::GetIsPrivate(bool* aPrivate) +{ + *aPrivate = IsPrivate(); + return NS_OK; +} + +NS_IMETHODIMP nsCacheSession::SetIsPrivate(bool aPrivate) +{ + if (aPrivate) + MarkPrivate(); + else + MarkPublic(); + return NS_OK; +} diff --git a/netwerk/cache/nsCacheSession.h b/netwerk/cache/nsCacheSession.h new file mode 100644 index 000000000..04b031902 --- /dev/null +++ b/netwerk/cache/nsCacheSession.h @@ -0,0 +1,67 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef _nsCacheSession_h_ +#define _nsCacheSession_h_ + +#include "nspr.h" +#include "nsError.h" +#include "nsCOMPtr.h" +#include "nsICacheSession.h" +#include "nsIFile.h" +#include "nsString.h" + +class nsCacheSession : public nsICacheSession +{ + virtual ~nsCacheSession(); + +public: + NS_DECL_ISUPPORTS + NS_DECL_NSICACHESESSION + + nsCacheSession(const char * clientID, nsCacheStoragePolicy storagePolicy, bool streamBased); + + nsCString * ClientID() { return &mClientID; } + + enum SessionInfo { + eStoragePolicyMask = 0x000000FF, + eStreamBasedMask = 0x00000100, + eDoomEntriesIfExpiredMask = 0x00001000, + ePrivateMask = 0x00010000 + }; + + void MarkStreamBased() { mInfo |= eStreamBasedMask; } + void ClearStreamBased() { mInfo &= ~eStreamBasedMask; } + bool IsStreamBased() { return (mInfo & eStreamBasedMask) != 0; } + + void MarkDoomEntriesIfExpired() { mInfo |= eDoomEntriesIfExpiredMask; } + void ClearDoomEntriesIfExpired() { mInfo &= ~eDoomEntriesIfExpiredMask; } + bool WillDoomEntriesIfExpired() { return (0 != (mInfo & eDoomEntriesIfExpiredMask)); } + + void MarkPrivate() { mInfo |= ePrivateMask; } + void MarkPublic() { mInfo &= ~ePrivateMask; } + bool IsPrivate() { return (mInfo & ePrivateMask) != 0; } + nsCacheStoragePolicy StoragePolicy() + { + return (nsCacheStoragePolicy)(mInfo & eStoragePolicyMask); + } + + void SetStoragePolicy(nsCacheStoragePolicy policy) + { + NS_ASSERTION(policy <= 0xFF, "too many bits in nsCacheStoragePolicy"); + mInfo &= ~eStoragePolicyMask; // clear storage policy bits + mInfo |= policy; + } + + nsIFile* ProfileDir() { return mProfileDir; } + +private: + nsCString mClientID; + uint32_t mInfo; + nsCOMPtr<nsIFile> mProfileDir; +}; + +#endif // _nsCacheSession_h_ diff --git a/netwerk/cache/nsCacheUtils.cpp b/netwerk/cache/nsCacheUtils.cpp new file mode 100644 index 000000000..7cb562274 --- /dev/null +++ b/netwerk/cache/nsCacheUtils.cpp @@ -0,0 +1,89 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsCache.h" +#include "nsCacheUtils.h" +#include "nsThreadUtils.h" + +using namespace mozilla; + +class nsDestroyThreadEvent : public Runnable { +public: + explicit nsDestroyThreadEvent(nsIThread *thread) + : mThread(thread) + {} + NS_IMETHOD Run() override + { + mThread->Shutdown(); + return NS_OK; + } +private: + nsCOMPtr<nsIThread> mThread; +}; + +nsShutdownThread::nsShutdownThread(nsIThread *aThread) + : mMonitor("nsShutdownThread.mMonitor") + , mShuttingDown(false) + , mThread(aThread) +{ +} + +nsShutdownThread::~nsShutdownThread() +{ +} + +nsresult +nsShutdownThread::Shutdown(nsIThread *aThread) +{ + nsresult rv; + RefPtr<nsDestroyThreadEvent> ev = new nsDestroyThreadEvent(aThread); + rv = NS_DispatchToMainThread(ev); + if (NS_FAILED(rv)) { + NS_WARNING("Dispatching event in nsShutdownThread::Shutdown failed!"); + } + return rv; +} + +nsresult +nsShutdownThread::BlockingShutdown(nsIThread *aThread) +{ + nsresult rv; + + RefPtr<nsShutdownThread> st = new nsShutdownThread(aThread); + nsCOMPtr<nsIThread> workerThread; + + rv = NS_NewNamedThread("thread shutdown", getter_AddRefs(workerThread)); + if (NS_FAILED(rv)) { + NS_WARNING("Can't create nsShutDownThread worker thread!"); + return rv; + } + + { + MonitorAutoLock mon(st->mMonitor); + rv = workerThread->Dispatch(st, NS_DISPATCH_NORMAL); + if (NS_FAILED(rv)) { + NS_WARNING( + "Dispatching event in nsShutdownThread::BlockingShutdown failed!"); + } else { + st->mShuttingDown = true; + while (st->mShuttingDown) { + mon.Wait(); + } + } + } + + return Shutdown(workerThread); +} + +NS_IMETHODIMP +nsShutdownThread::Run() +{ + MonitorAutoLock mon(mMonitor); + mThread->Shutdown(); + mShuttingDown = false; + mon.Notify(); + return NS_OK; +} diff --git a/netwerk/cache/nsCacheUtils.h b/netwerk/cache/nsCacheUtils.h new file mode 100644 index 000000000..65ae8f82b --- /dev/null +++ b/netwerk/cache/nsCacheUtils.h @@ -0,0 +1,43 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* vim: set sw=2 ts=8 et tw=80 : */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef _nsCacheUtils_h_ +#define _nsCacheUtils_h_ + +#include "nsThreadUtils.h" +#include "nsCOMPtr.h" +#include "mozilla/Monitor.h" + +class nsIThread; + +/** + * A class with utility methods for shutting down nsIThreads easily. + */ +class nsShutdownThread : public mozilla::Runnable { +public: + explicit nsShutdownThread(nsIThread *aThread); + ~nsShutdownThread(); + + NS_IMETHOD Run(); + +/** + * Shutdown ensures that aThread->Shutdown() is called on a main thread + */ + static nsresult Shutdown(nsIThread *aThread); + +/** + * BlockingShutdown ensures that by the time it returns, aThread->Shutdown() has + * been called and no pending events have been processed on the current thread. + */ + static nsresult BlockingShutdown(nsIThread *aThread); + +private: + mozilla::Monitor mMonitor; + bool mShuttingDown; + nsCOMPtr<nsIThread> mThread; +}; + +#endif diff --git a/netwerk/cache/nsDeleteDir.cpp b/netwerk/cache/nsDeleteDir.cpp new file mode 100644 index 000000000..1f3f3934e --- /dev/null +++ b/netwerk/cache/nsDeleteDir.cpp @@ -0,0 +1,454 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsDeleteDir.h" +#include "nsIFile.h" +#include "nsString.h" +#include "mozilla/Telemetry.h" +#include "nsITimer.h" +#include "nsISimpleEnumerator.h" +#include "nsAutoPtr.h" +#include "nsThreadUtils.h" +#include "nsISupportsPriority.h" +#include "nsCacheUtils.h" +#include "prtime.h" +#include <time.h> + +using namespace mozilla; + +class nsBlockOnBackgroundThreadEvent : public Runnable { +public: + nsBlockOnBackgroundThreadEvent() {} + NS_IMETHOD Run() override + { + MutexAutoLock lock(nsDeleteDir::gInstance->mLock); + nsDeleteDir::gInstance->mNotified = true; + nsDeleteDir::gInstance->mCondVar.Notify(); + return NS_OK; + } +}; + + +nsDeleteDir * nsDeleteDir::gInstance = nullptr; + +nsDeleteDir::nsDeleteDir() + : mLock("nsDeleteDir.mLock"), + mCondVar(mLock, "nsDeleteDir.mCondVar"), + mNotified(false), + mShutdownPending(false), + mStopDeleting(false) +{ + NS_ASSERTION(gInstance==nullptr, "multiple nsCacheService instances!"); +} + +nsDeleteDir::~nsDeleteDir() +{ + gInstance = nullptr; +} + +nsresult +nsDeleteDir::Init() +{ + if (gInstance) + return NS_ERROR_ALREADY_INITIALIZED; + + gInstance = new nsDeleteDir(); + return NS_OK; +} + +nsresult +nsDeleteDir::Shutdown(bool finishDeleting) +{ + if (!gInstance) + return NS_ERROR_NOT_INITIALIZED; + + nsCOMArray<nsIFile> dirsToRemove; + nsCOMPtr<nsIThread> thread; + { + MutexAutoLock lock(gInstance->mLock); + NS_ASSERTION(!gInstance->mShutdownPending, + "Unexpected state in nsDeleteDir::Shutdown()"); + gInstance->mShutdownPending = true; + + if (!finishDeleting) + gInstance->mStopDeleting = true; + + // remove all pending timers + for (int32_t i = gInstance->mTimers.Count(); i > 0; i--) { + nsCOMPtr<nsITimer> timer = gInstance->mTimers[i-1]; + gInstance->mTimers.RemoveObjectAt(i-1); + + nsCOMArray<nsIFile> *arg; + timer->GetClosure((reinterpret_cast<void**>(&arg))); + timer->Cancel(); + + if (finishDeleting) + dirsToRemove.AppendObjects(*arg); + + // delete argument passed to the timer + delete arg; + } + + thread.swap(gInstance->mThread); + if (thread) { + // dispatch event and wait for it to run and notify us, so we know thread + // has completed all work and can be shutdown + nsCOMPtr<nsIRunnable> event = new nsBlockOnBackgroundThreadEvent(); + nsresult rv = thread->Dispatch(event, NS_DISPATCH_NORMAL); + if (NS_FAILED(rv)) { + NS_WARNING("Failed dispatching block-event"); + return NS_ERROR_UNEXPECTED; + } + + gInstance->mNotified = false; + while (!gInstance->mNotified) { + gInstance->mCondVar.Wait(); + } + nsShutdownThread::BlockingShutdown(thread); + } + } + + delete gInstance; + + for (int32_t i = 0; i < dirsToRemove.Count(); i++) + dirsToRemove[i]->Remove(true); + + return NS_OK; +} + +nsresult +nsDeleteDir::InitThread() +{ + if (mThread) + return NS_OK; + + nsresult rv = NS_NewNamedThread("Cache Deleter", getter_AddRefs(mThread)); + if (NS_FAILED(rv)) { + NS_WARNING("Can't create background thread"); + return rv; + } + + nsCOMPtr<nsISupportsPriority> p = do_QueryInterface(mThread); + if (p) { + p->SetPriority(nsISupportsPriority::PRIORITY_LOWEST); + } + return NS_OK; +} + +void +nsDeleteDir::DestroyThread() +{ + if (!mThread) + return; + + if (mTimers.Count()) + // more work to do, so don't delete thread. + return; + + nsShutdownThread::Shutdown(mThread); + mThread = nullptr; +} + +void +nsDeleteDir::TimerCallback(nsITimer *aTimer, void *arg) +{ + Telemetry::AutoTimer<Telemetry::NETWORK_DISK_CACHE_DELETEDIR> timer; + { + MutexAutoLock lock(gInstance->mLock); + + int32_t idx = gInstance->mTimers.IndexOf(aTimer); + if (idx == -1) { + // Timer was canceled and removed during shutdown. + return; + } + + gInstance->mTimers.RemoveObjectAt(idx); + } + + nsAutoPtr<nsCOMArray<nsIFile> > dirList; + dirList = static_cast<nsCOMArray<nsIFile> *>(arg); + + bool shuttingDown = false; + + // Intentional extra braces to control variable sope. + { + // Low IO priority can only be set when running in the context of the + // current thread. So this shouldn't be moved to where we set the priority + // of the Cache deleter thread using the nsThread's NSPR priority constants. + nsAutoLowPriorityIO autoLowPriority; + for (int32_t i = 0; i < dirList->Count() && !shuttingDown; i++) { + gInstance->RemoveDir((*dirList)[i], &shuttingDown); + } + } + + { + MutexAutoLock lock(gInstance->mLock); + gInstance->DestroyThread(); + } +} + +nsresult +nsDeleteDir::DeleteDir(nsIFile *dirIn, bool moveToTrash, uint32_t delay) +{ + Telemetry::AutoTimer<Telemetry::NETWORK_DISK_CACHE_TRASHRENAME> timer; + + if (!gInstance) + return NS_ERROR_NOT_INITIALIZED; + + nsresult rv; + nsCOMPtr<nsIFile> trash, dir; + + // Need to make a clone of this since we don't want to modify the input + // file object. + rv = dirIn->Clone(getter_AddRefs(dir)); + if (NS_FAILED(rv)) + return rv; + + if (moveToTrash) { + rv = GetTrashDir(dir, &trash); + if (NS_FAILED(rv)) + return rv; + nsAutoCString origLeaf; + rv = trash->GetNativeLeafName(origLeaf); + if (NS_FAILED(rv)) + return rv; + + // Append random number to the trash directory and check if it exists. + srand(static_cast<unsigned>(PR_Now())); + nsAutoCString leaf; + for (int32_t i = 0; i < 10; i++) { + leaf = origLeaf; + leaf.AppendInt(rand()); + rv = trash->SetNativeLeafName(leaf); + if (NS_FAILED(rv)) + return rv; + + bool exists; + if (NS_SUCCEEDED(trash->Exists(&exists)) && !exists) { + break; + } + + leaf.Truncate(); + } + + // Fail if we didn't find unused trash directory within the limit + if (!leaf.Length()) + return NS_ERROR_FAILURE; + +#if defined(MOZ_WIDGET_ANDROID) + nsCOMPtr<nsIFile> parent; + rv = trash->GetParent(getter_AddRefs(parent)); + if (NS_FAILED(rv)) + return rv; + rv = dir->MoveToNative(parent, leaf); +#else + // Important: must rename directory w/o changing parent directory: else on + // NTFS we'll wait (with cache lock) while nsIFile's ACL reset walks file + // tree: was hanging GUI for *minutes* on large cache dirs. + rv = dir->MoveToNative(nullptr, leaf); +#endif + if (NS_FAILED(rv)) + return rv; + } else { + // we want to pass a clone of the original off to the worker thread. + trash.swap(dir); + } + + nsAutoPtr<nsCOMArray<nsIFile> > arg(new nsCOMArray<nsIFile>); + arg->AppendObject(trash); + + rv = gInstance->PostTimer(arg, delay); + if (NS_FAILED(rv)) + return rv; + + arg.forget(); + return NS_OK; +} + +nsresult +nsDeleteDir::GetTrashDir(nsIFile *target, nsCOMPtr<nsIFile> *result) +{ + nsresult rv; +#if defined(MOZ_WIDGET_ANDROID) + // Try to use the app cache folder for cache trash on Android + char* cachePath = getenv("CACHE_DIRECTORY"); + if (cachePath) { + rv = NS_NewNativeLocalFile(nsDependentCString(cachePath), + true, getter_AddRefs(*result)); + if (NS_FAILED(rv)) + return rv; + + // Add a sub folder with the cache folder name + nsAutoCString leaf; + rv = target->GetNativeLeafName(leaf); + (*result)->AppendNative(leaf); + } else +#endif + { + rv = target->Clone(getter_AddRefs(*result)); + } + if (NS_FAILED(rv)) + return rv; + + nsAutoCString leaf; + rv = (*result)->GetNativeLeafName(leaf); + if (NS_FAILED(rv)) + return rv; + leaf.AppendLiteral(".Trash"); + + return (*result)->SetNativeLeafName(leaf); +} + +nsresult +nsDeleteDir::RemoveOldTrashes(nsIFile *cacheDir) +{ + if (!gInstance) + return NS_ERROR_NOT_INITIALIZED; + + nsresult rv; + + nsCOMPtr<nsIFile> trash; + rv = GetTrashDir(cacheDir, &trash); + if (NS_FAILED(rv)) + return rv; + + nsAutoString trashName; + rv = trash->GetLeafName(trashName); + if (NS_FAILED(rv)) + return rv; + + nsCOMPtr<nsIFile> parent; +#if defined(MOZ_WIDGET_ANDROID) + rv = trash->GetParent(getter_AddRefs(parent)); +#else + rv = cacheDir->GetParent(getter_AddRefs(parent)); +#endif + if (NS_FAILED(rv)) + return rv; + + nsCOMPtr<nsISimpleEnumerator> iter; + rv = parent->GetDirectoryEntries(getter_AddRefs(iter)); + if (NS_FAILED(rv)) + return rv; + + bool more; + nsCOMPtr<nsISupports> elem; + nsAutoPtr<nsCOMArray<nsIFile> > dirList; + + while (NS_SUCCEEDED(iter->HasMoreElements(&more)) && more) { + rv = iter->GetNext(getter_AddRefs(elem)); + if (NS_FAILED(rv)) + continue; + + nsCOMPtr<nsIFile> file = do_QueryInterface(elem); + if (!file) + continue; + + nsAutoString leafName; + rv = file->GetLeafName(leafName); + if (NS_FAILED(rv)) + continue; + + // match all names that begin with the trash name (i.e. "Cache.Trash") + if (Substring(leafName, 0, trashName.Length()).Equals(trashName)) { + if (!dirList) + dirList = new nsCOMArray<nsIFile>; + dirList->AppendObject(file); + } + } + + if (dirList) { + rv = gInstance->PostTimer(dirList, 90000); + if (NS_FAILED(rv)) + return rv; + + dirList.forget(); + } + + return NS_OK; +} + +nsresult +nsDeleteDir::PostTimer(void *arg, uint32_t delay) +{ + nsresult rv; + + nsCOMPtr<nsITimer> timer = do_CreateInstance("@mozilla.org/timer;1", &rv); + if (NS_FAILED(rv)) + return NS_ERROR_UNEXPECTED; + + MutexAutoLock lock(mLock); + + rv = InitThread(); + if (NS_FAILED(rv)) + return rv; + + rv = timer->SetTarget(mThread); + if (NS_FAILED(rv)) + return rv; + + rv = timer->InitWithFuncCallback(TimerCallback, arg, delay, + nsITimer::TYPE_ONE_SHOT); + if (NS_FAILED(rv)) + return rv; + + mTimers.AppendObject(timer); + return NS_OK; +} + +nsresult +nsDeleteDir::RemoveDir(nsIFile *file, bool *stopDeleting) +{ + nsresult rv; + bool isLink; + + rv = file->IsSymlink(&isLink); + if (NS_FAILED(rv) || isLink) + return NS_ERROR_UNEXPECTED; + + bool isDir; + rv = file->IsDirectory(&isDir); + if (NS_FAILED(rv)) + return rv; + + if (isDir) { + nsCOMPtr<nsISimpleEnumerator> iter; + rv = file->GetDirectoryEntries(getter_AddRefs(iter)); + if (NS_FAILED(rv)) + return rv; + + bool more; + nsCOMPtr<nsISupports> elem; + while (NS_SUCCEEDED(iter->HasMoreElements(&more)) && more) { + rv = iter->GetNext(getter_AddRefs(elem)); + if (NS_FAILED(rv)) { + NS_WARNING("Unexpected failure in nsDeleteDir::RemoveDir"); + continue; + } + + nsCOMPtr<nsIFile> file2 = do_QueryInterface(elem); + if (!file2) { + NS_WARNING("Unexpected failure in nsDeleteDir::RemoveDir"); + continue; + } + + RemoveDir(file2, stopDeleting); + // No check for errors to remove as much as possible + + if (*stopDeleting) + return NS_OK; + } + } + + file->Remove(false); + // No check for errors to remove as much as possible + + MutexAutoLock lock(mLock); + if (mStopDeleting) + *stopDeleting = true; + + return NS_OK; +} diff --git a/netwerk/cache/nsDeleteDir.h b/netwerk/cache/nsDeleteDir.h new file mode 100644 index 000000000..6426efd26 --- /dev/null +++ b/netwerk/cache/nsDeleteDir.h @@ -0,0 +1,80 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cindent: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef nsDeleteDir_h__ +#define nsDeleteDir_h__ + +#include "nsCOMPtr.h" +#include "nsCOMArray.h" +#include "mozilla/Mutex.h" +#include "mozilla/CondVar.h" + +class nsIFile; +class nsIThread; +class nsITimer; + + +class nsDeleteDir { +public: + nsDeleteDir(); + ~nsDeleteDir(); + + static nsresult Init(); + static nsresult Shutdown(bool finishDeleting); + + /** + * This routine attempts to delete a directory that may contain some files + * that are still in use. This latter point is only an issue on Windows and + * a few other systems. + * + * If the moveToTrash parameter is true we first rename the given directory + * "foo.Trash123" (where "foo" is the original directory name, and "123" is + * a random number, in order to support multiple concurrent deletes). The + * directory is then deleted, file-by-file, on a background thread. + * + * If the moveToTrash parameter is false, then the given directory is deleted + * directly. + * + * If 'delay' is non-zero, the directory will not be deleted until the + * specified number of milliseconds have passed. (The directory is still + * renamed immediately if 'moveToTrash' is passed, so upon return it is safe + * to create a directory with the same name). + */ + static nsresult DeleteDir(nsIFile *dir, bool moveToTrash, uint32_t delay = 0); + + /** + * Returns the trash directory corresponding to the given directory. + */ + static nsresult GetTrashDir(nsIFile *dir, nsCOMPtr<nsIFile> *result); + + /** + * Remove all trashes left from previous run. This function does nothing when + * called second and more times. + */ + static nsresult RemoveOldTrashes(nsIFile *cacheDir); + + static void TimerCallback(nsITimer *aTimer, void *arg); + +private: + friend class nsBlockOnBackgroundThreadEvent; + friend class nsDestroyThreadEvent; + + nsresult InitThread(); + void DestroyThread(); + nsresult PostTimer(void *arg, uint32_t delay); + nsresult RemoveDir(nsIFile *file, bool *stopDeleting); + + static nsDeleteDir * gInstance; + mozilla::Mutex mLock; + mozilla::CondVar mCondVar; + bool mNotified; + nsCOMArray<nsITimer> mTimers; + nsCOMPtr<nsIThread> mThread; + bool mShutdownPending; + bool mStopDeleting; +}; + +#endif // nsDeleteDir_h__ diff --git a/netwerk/cache/nsDiskCache.h b/netwerk/cache/nsDiskCache.h new file mode 100644 index 000000000..cc91553fa --- /dev/null +++ b/netwerk/cache/nsDiskCache.h @@ -0,0 +1,72 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + + +#ifndef _nsDiskCache_h_ +#define _nsDiskCache_h_ + +#include "nsCacheEntry.h" + +#ifdef XP_WIN +#include <winsock.h> // for htonl/ntohl +#endif + + +class nsDiskCache { +public: + enum { + kCurrentVersion = 0x00010013 // format = 16 bits major version/16 bits minor version + }; + + enum { kData, kMetaData }; + + // Stores the reason why the cache is corrupt. + // Note: I'm only listing the enum values explicitly for easy mapping when + // looking at telemetry data. + enum CorruptCacheInfo { + kNotCorrupt = 0, + kInvalidArgPointer = 1, + kUnexpectedError = 2, + kOpenCacheMapError = 3, + kBlockFilesShouldNotExist = 4, + kOutOfMemory = 5, + kCreateCacheSubdirectories = 6, + kBlockFilesShouldExist = 7, + kHeaderSizeNotRead = 8, + kHeaderIsDirty = 9, + kVersionMismatch = 10, + kRecordsIncomplete = 11, + kHeaderIncomplete = 12, + kNotEnoughToRead = 13, + kEntryCountIncorrect = 14, + kCouldNotGetBlockFileForIndex = 15, + kCouldNotCreateBlockFile = 16, + kBlockFileSizeError = 17, + kBlockFileBitMapWriteError = 18, + kBlockFileSizeLessThanBitMap = 19, + kBlockFileBitMapReadError = 20, + kBlockFileEstimatedSizeError = 21, + kFlushHeaderError = 22, + kCacheCleanFilePathError = 23, + kCacheCleanOpenFileError = 24, + kCacheCleanTimerError = 25 + }; + + // Parameter initval initializes internal state of hash function. Hash values are different + // for the same text when different initval is used. It can be any random number. + // + // It can be used for generating 64-bit hash value: + // (uint64_t(Hash(key, initval1)) << 32) | Hash(key, initval2) + // + // It can be also used to hash multiple strings: + // h = Hash(string1, 0); + // h = Hash(string2, h); + // ... + static PLDHashNumber Hash(const char* key, PLDHashNumber initval=0); + static nsresult Truncate(PRFileDesc * fd, uint32_t newEOF); +}; + +#endif // _nsDiskCache_h_ diff --git a/netwerk/cache/nsDiskCacheBinding.cpp b/netwerk/cache/nsDiskCacheBinding.cpp new file mode 100644 index 000000000..cdc2606e6 --- /dev/null +++ b/netwerk/cache/nsDiskCacheBinding.cpp @@ -0,0 +1,371 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "mozilla/MemoryReporting.h" +#include "nsCache.h" +#include <limits.h> + +#include "nscore.h" +#include "nsDiskCacheBinding.h" +#include "nsCacheService.h" + +using namespace mozilla; + +/****************************************************************************** + * static hash table callback functions + * + *****************************************************************************/ +struct HashTableEntry : PLDHashEntryHdr { + nsDiskCacheBinding * mBinding; +}; + + +static PLDHashNumber +HashKey(const void *key) +{ + return (PLDHashNumber) NS_PTR_TO_INT32(key); +} + + +static bool +MatchEntry(const PLDHashEntryHdr * header, + const void * key) +{ + HashTableEntry * hashEntry = (HashTableEntry *) header; + return (hashEntry->mBinding->mRecord.HashNumber() == (PLDHashNumber) NS_PTR_TO_INT32(key)); +} + +static void +MoveEntry(PLDHashTable * /* table */, + const PLDHashEntryHdr * src, + PLDHashEntryHdr * dst) +{ + ((HashTableEntry *)dst)->mBinding = ((HashTableEntry *)src)->mBinding; +} + + +static void +ClearEntry(PLDHashTable * /* table */, + PLDHashEntryHdr * header) +{ + ((HashTableEntry *)header)->mBinding = nullptr; +} + + +/****************************************************************************** + * Utility Functions + *****************************************************************************/ +nsDiskCacheBinding * +GetCacheEntryBinding(nsCacheEntry * entry) +{ + return (nsDiskCacheBinding *) entry->Data(); +} + + +/****************************************************************************** + * nsDiskCacheBinding + *****************************************************************************/ + +NS_IMPL_ISUPPORTS0(nsDiskCacheBinding) + +nsDiskCacheBinding::nsDiskCacheBinding(nsCacheEntry* entry, nsDiskCacheRecord * record) + : mCacheEntry(entry) + , mStreamIO(nullptr) + , mDeactivateEvent(nullptr) +{ + NS_ASSERTION(record->ValidRecord(), "bad record"); + PR_INIT_CLIST(this); + mRecord = *record; + mDoomed = entry->IsDoomed(); + mGeneration = record->Generation(); // 0 == uninitialized, or data & meta using block files +} + +nsDiskCacheBinding::~nsDiskCacheBinding() +{ + // Grab the cache lock since the binding is stored in nsCacheEntry::mData + // and it is released using nsCacheService::ReleaseObject_Locked() which + // releases the object outside the cache lock. + nsCacheServiceAutoLock lock; + + NS_ASSERTION(PR_CLIST_IS_EMPTY(this), "binding deleted while still on list"); + if (!PR_CLIST_IS_EMPTY(this)) + PR_REMOVE_LINK(this); // XXX why are we still on a list? + + // sever streamIO/binding link + if (mStreamIO) { + if (NS_FAILED(mStreamIO->ClearBinding())) + nsCacheService::DoomEntry(mCacheEntry); + NS_RELEASE(mStreamIO); + } +} + +nsresult +nsDiskCacheBinding::EnsureStreamIO() +{ + if (!mStreamIO) { + mStreamIO = new nsDiskCacheStreamIO(this); + if (!mStreamIO) return NS_ERROR_OUT_OF_MEMORY; + NS_ADDREF(mStreamIO); + } + return NS_OK; +} + + +/****************************************************************************** + * nsDiskCacheBindery + * + * Keeps track of bound disk cache entries to detect for collisions. + * + *****************************************************************************/ + +const PLDHashTableOps nsDiskCacheBindery::ops = +{ + HashKey, + MatchEntry, + MoveEntry, + ClearEntry +}; + + +nsDiskCacheBindery::nsDiskCacheBindery() + : table(&ops, sizeof(HashTableEntry), kInitialTableLength) + , initialized(false) +{ +} + + +nsDiskCacheBindery::~nsDiskCacheBindery() +{ + Reset(); +} + + +void +nsDiskCacheBindery::Init() +{ + table.ClearAndPrepareForLength(kInitialTableLength); + initialized = true; +} + +void +nsDiskCacheBindery::Reset() +{ + if (initialized) { + table.ClearAndPrepareForLength(kInitialTableLength); + initialized = false; + } +} + + +nsDiskCacheBinding * +nsDiskCacheBindery::CreateBinding(nsCacheEntry * entry, + nsDiskCacheRecord * record) +{ + NS_ASSERTION(initialized, "nsDiskCacheBindery not initialized"); + nsCOMPtr<nsISupports> data = entry->Data(); + if (data) { + NS_ERROR("cache entry already has bind data"); + return nullptr; + } + + nsDiskCacheBinding * binding = new nsDiskCacheBinding(entry, record); + if (!binding) return nullptr; + + // give ownership of the binding to the entry + entry->SetData(binding); + + // add binding to collision detection system + nsresult rv = AddBinding(binding); + if (NS_FAILED(rv)) { + entry->SetData(nullptr); + return nullptr; + } + + return binding; +} + + +/** + * FindActiveEntry : to find active colliding entry so we can doom it + */ +nsDiskCacheBinding * +nsDiskCacheBindery::FindActiveBinding(uint32_t hashNumber) +{ + NS_ASSERTION(initialized, "nsDiskCacheBindery not initialized"); + // find hash entry for key + auto hashEntry = static_cast<HashTableEntry*> + (table.Search((void*)(uintptr_t)hashNumber)); + if (!hashEntry) return nullptr; + + // walk list looking for active entry + NS_ASSERTION(hashEntry->mBinding, "hash entry left with no binding"); + nsDiskCacheBinding * binding = hashEntry->mBinding; + while (binding->mCacheEntry->IsDoomed()) { + binding = (nsDiskCacheBinding *)PR_NEXT_LINK(binding); + if (binding == hashEntry->mBinding) return nullptr; + } + return binding; +} + + +/** + * AddBinding + * + * Called from FindEntry() if we read an entry off of disk + * - it may already have a generation number + * - a generation number conflict is an error + * + * Called from BindEntry() + * - a generation number needs to be assigned + */ +nsresult +nsDiskCacheBindery::AddBinding(nsDiskCacheBinding * binding) +{ + NS_ENSURE_ARG_POINTER(binding); + NS_ASSERTION(initialized, "nsDiskCacheBindery not initialized"); + + // find hash entry for key + auto hashEntry = static_cast<HashTableEntry*> + (table.Add((void*)(uintptr_t)binding->mRecord.HashNumber(), fallible)); + if (!hashEntry) + return NS_ERROR_OUT_OF_MEMORY; + + if (hashEntry->mBinding == nullptr) { + hashEntry->mBinding = binding; + if (binding->mGeneration == 0) + binding->mGeneration = 1; // if generation uninitialized, set it to 1 + + return NS_OK; + } + + + // insert binding in generation order + nsDiskCacheBinding * p = hashEntry->mBinding; + bool calcGeneration = (binding->mGeneration == 0); // do we need to calculate generation? + if (calcGeneration) binding->mGeneration = 1; // initialize to 1 if uninitialized + while (1) { + + if (binding->mGeneration < p->mGeneration) { + // here we are + PR_INSERT_BEFORE(binding, p); + if (hashEntry->mBinding == p) + hashEntry->mBinding = binding; + break; + } + + if (binding->mGeneration == p->mGeneration) { + if (calcGeneration) ++binding->mGeneration; // try the next generation + else { + NS_ERROR("### disk cache: generations collide!"); + return NS_ERROR_UNEXPECTED; + } + } + + p = (nsDiskCacheBinding *)PR_NEXT_LINK(p); + if (p == hashEntry->mBinding) { + // end of line: insert here or die + p = (nsDiskCacheBinding *)PR_PREV_LINK(p); // back up and check generation + if (p->mGeneration == 255) { + NS_WARNING("### disk cache: generation capacity at full"); + return NS_ERROR_UNEXPECTED; + } + PR_INSERT_BEFORE(binding, hashEntry->mBinding); + break; + } + } + return NS_OK; +} + + +/** + * RemoveBinding : remove binding from collision detection on deactivation + */ +void +nsDiskCacheBindery::RemoveBinding(nsDiskCacheBinding * binding) +{ + NS_ASSERTION(initialized, "nsDiskCacheBindery not initialized"); + if (!initialized) return; + + void* key = (void *)(uintptr_t)binding->mRecord.HashNumber(); + auto hashEntry = + static_cast<HashTableEntry*>(table.Search((void*)(uintptr_t) key)); + if (!hashEntry) { + NS_WARNING("### disk cache: binding not in hashtable!"); + return; + } + + if (binding == hashEntry->mBinding) { + if (PR_CLIST_IS_EMPTY(binding)) { + // remove this hash entry + table.Remove((void*)(uintptr_t) binding->mRecord.HashNumber()); + return; + + } else { + // promote next binding to head, and unlink this binding + hashEntry->mBinding = (nsDiskCacheBinding *)PR_NEXT_LINK(binding); + } + } + PR_REMOVE_AND_INIT_LINK(binding); +} + +/** + * ActiveBindings: return true if any bindings have open descriptors. + */ +bool +nsDiskCacheBindery::ActiveBindings() +{ + NS_ASSERTION(initialized, "nsDiskCacheBindery not initialized"); + if (!initialized) return false; + + for (auto iter = table.Iter(); !iter.Done(); iter.Next()) { + auto entry = static_cast<HashTableEntry*>(iter.Get()); + nsDiskCacheBinding* binding = entry->mBinding; + nsDiskCacheBinding* head = binding; + do { + if (binding->IsActive()) { + return true; + } + binding = (nsDiskCacheBinding *)PR_NEXT_LINK(binding); + } while (binding != head); + } + + return false; +} + +/** + * SizeOfExcludingThis: return the amount of heap memory (bytes) being used by + * the bindery. + */ +size_t +nsDiskCacheBindery::SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf) +{ + NS_ASSERTION(initialized, "nsDiskCacheBindery not initialized"); + if (!initialized) return 0; + + size_t size = 0; + + for (auto iter = table.Iter(); !iter.Done(); iter.Next()) { + auto entry = static_cast<HashTableEntry*>(iter.Get()); + nsDiskCacheBinding* binding = entry->mBinding; + + nsDiskCacheBinding* head = binding; + do { + size += aMallocSizeOf(binding); + if (binding->mStreamIO) { + size += binding->mStreamIO->SizeOfIncludingThis(aMallocSizeOf); + } + + // No good way to get at mDeactivateEvent internals for proper + // size, so we use this as an estimate. + if (binding->mDeactivateEvent) { + size += aMallocSizeOf(binding->mDeactivateEvent); + } + binding = (nsDiskCacheBinding *)PR_NEXT_LINK(binding); + } while (binding != head); + } + + return size; +} diff --git a/netwerk/cache/nsDiskCacheBinding.h b/netwerk/cache/nsDiskCacheBinding.h new file mode 100644 index 000000000..63c2094ce --- /dev/null +++ b/netwerk/cache/nsDiskCacheBinding.h @@ -0,0 +1,123 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + + +#ifndef _nsDiskCacheBinding_h_ +#define _nsDiskCacheBinding_h_ + +#include "mozilla/MemoryReporting.h" +#include "nspr.h" +#include "PLDHashTable.h" + +#include "nsISupports.h" +#include "nsCacheEntry.h" + +#include "nsDiskCacheMap.h" +#include "nsDiskCacheStreams.h" + + +/****************************************************************************** + * nsDiskCacheBinding + * + * Created for disk cache specific data and stored in nsCacheEntry.mData as + * an nsISupports. Also stored in nsDiskCacheHashTable, with collisions + * linked by the PRCList. + * + *****************************************************************************/ + +class nsDiskCacheDeviceDeactivateEntryEvent; + +class nsDiskCacheBinding : public nsISupports, public PRCList { + virtual ~nsDiskCacheBinding(); + +public: + NS_DECL_THREADSAFE_ISUPPORTS + + nsDiskCacheBinding(nsCacheEntry* entry, nsDiskCacheRecord * record); + + nsresult EnsureStreamIO(); + bool IsActive() { return mCacheEntry != nullptr;} + +// XXX make friends +public: + nsCacheEntry* mCacheEntry; // back pointer to parent nsCacheEntry + nsDiskCacheRecord mRecord; + nsDiskCacheStreamIO* mStreamIO; // strong reference + bool mDoomed; // record is not stored in cache map + uint8_t mGeneration; // possibly just reservation + + // If set, points to a pending event which will deactivate |mCacheEntry|. + // If not set then either |mCacheEntry| is not deactivated, or it has been + // deactivated but the device returned it from FindEntry() before the event + // fired. In both two latter cases this binding is to be considered valid. + nsDiskCacheDeviceDeactivateEntryEvent *mDeactivateEvent; +}; + + +/****************************************************************************** + * Utility Functions + *****************************************************************************/ + +nsDiskCacheBinding * GetCacheEntryBinding(nsCacheEntry * entry); + + + +/****************************************************************************** + * nsDiskCacheBindery + * + * Used to keep track of nsDiskCacheBinding associated with active/bound (and + * possibly doomed) entries. Lookups on 4 byte disk hash to find collisions + * (which need to be doomed, instead of just evicted. Collisions are linked + * using a PRCList to keep track of current generation number. + * + * Used to detect hash number collisions, and find available generation numbers. + * + * Not all nsDiskCacheBinding have a generation number. + * + * Generation numbers may be aquired late, or lost (when data fits in block file) + * + * Collisions can occur: + * BindEntry() - hashnumbers collide (possibly different keys) + * + * Generation number required: + * DeactivateEntry() - metadata written to disk, may require file + * GetFileForEntry() - force data to require file + * writing to stream - data size may require file + * + * Binding can be kept in PRCList in order of generation numbers. + * Binding with no generation number can be Appended to PRCList (last). + * + *****************************************************************************/ + +class nsDiskCacheBindery { +public: + nsDiskCacheBindery(); + ~nsDiskCacheBindery(); + + void Init(); + void Reset(); + + nsDiskCacheBinding * CreateBinding(nsCacheEntry * entry, + nsDiskCacheRecord * record); + + nsDiskCacheBinding * FindActiveBinding(uint32_t hashNumber); + void RemoveBinding(nsDiskCacheBinding * binding); + bool ActiveBindings(); + + size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf); + +private: + nsresult AddBinding(nsDiskCacheBinding * binding); + + // member variables + static const PLDHashTableOps ops; + PLDHashTable table; + bool initialized; + + static const uint32_t kInitialTableLength = 0; +}; + +#endif /* _nsDiskCacheBinding_h_ */ diff --git a/netwerk/cache/nsDiskCacheBlockFile.cpp b/netwerk/cache/nsDiskCacheBlockFile.cpp new file mode 100644 index 000000000..6a1f4182f --- /dev/null +++ b/netwerk/cache/nsDiskCacheBlockFile.cpp @@ -0,0 +1,404 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsCache.h" +#include "nsDiskCache.h" +#include "nsDiskCacheBlockFile.h" +#include "mozilla/FileUtils.h" +#include "mozilla/MemoryReporting.h" +#include <algorithm> + +using namespace mozilla; + +/****************************************************************************** + * nsDiskCacheBlockFile - + *****************************************************************************/ + +/****************************************************************************** + * Open + *****************************************************************************/ +nsresult +nsDiskCacheBlockFile::Open(nsIFile * blockFile, + uint32_t blockSize, + uint32_t bitMapSize, + nsDiskCache::CorruptCacheInfo * corruptInfo) +{ + NS_ENSURE_ARG_POINTER(corruptInfo); + *corruptInfo = nsDiskCache::kUnexpectedError; + + if (bitMapSize % 32) { + *corruptInfo = nsDiskCache::kInvalidArgPointer; + return NS_ERROR_INVALID_ARG; + } + + mBlockSize = blockSize; + mBitMapWords = bitMapSize / 32; + uint32_t bitMapBytes = mBitMapWords * 4; + + // open the file - restricted to user, the data could be confidential + nsresult rv = blockFile->OpenNSPRFileDesc(PR_RDWR | PR_CREATE_FILE, 00600, &mFD); + if (NS_FAILED(rv)) { + *corruptInfo = nsDiskCache::kCouldNotCreateBlockFile; + CACHE_LOG_DEBUG(("CACHE: nsDiskCacheBlockFile::Open " + "[this=%p] unable to open or create file: %d", + this, rv)); + return rv; // unable to open or create file + } + + // allocate bit map buffer + mBitMap = new uint32_t[mBitMapWords]; + + // check if we just creating the file + mFileSize = PR_Available(mFD); + if (mFileSize < 0) { + // XXX an error occurred. We could call PR_GetError(), but how would that help? + *corruptInfo = nsDiskCache::kBlockFileSizeError; + rv = NS_ERROR_UNEXPECTED; + goto error_exit; + } + if (mFileSize == 0) { + // initialize bit map and write it + memset(mBitMap, 0, bitMapBytes); + if (!Write(0, mBitMap, bitMapBytes)) { + *corruptInfo = nsDiskCache::kBlockFileBitMapWriteError; + goto error_exit; + } + + } else if ((uint32_t)mFileSize < bitMapBytes) { + *corruptInfo = nsDiskCache::kBlockFileSizeLessThanBitMap; + rv = NS_ERROR_UNEXPECTED; // XXX NS_ERROR_CACHE_INVALID; + goto error_exit; + + } else { + // read the bit map + const int32_t bytesRead = PR_Read(mFD, mBitMap, bitMapBytes); + if ((bytesRead < 0) || ((uint32_t)bytesRead < bitMapBytes)) { + *corruptInfo = nsDiskCache::kBlockFileBitMapReadError; + rv = NS_ERROR_UNEXPECTED; + goto error_exit; + } +#if defined(IS_LITTLE_ENDIAN) + // Swap from network format + for (unsigned int i = 0; i < mBitMapWords; ++i) + mBitMap[i] = ntohl(mBitMap[i]); +#endif + // validate block file size + // Because not whole blocks are written, the size may be a + // little bit smaller than used blocks times blocksize, + // because the last block will generally not be 'whole'. + const uint32_t estimatedSize = CalcBlockFileSize(); + if ((uint32_t)mFileSize + blockSize < estimatedSize) { + *corruptInfo = nsDiskCache::kBlockFileEstimatedSizeError; + rv = NS_ERROR_UNEXPECTED; + goto error_exit; + } + } + CACHE_LOG_DEBUG(("CACHE: nsDiskCacheBlockFile::Open [this=%p] succeeded", + this)); + return NS_OK; + +error_exit: + CACHE_LOG_DEBUG(("CACHE: nsDiskCacheBlockFile::Open [this=%p] failed with " + "error %d", this, rv)); + Close(false); + return rv; +} + + +/****************************************************************************** + * Close + *****************************************************************************/ +nsresult +nsDiskCacheBlockFile::Close(bool flush) +{ + nsresult rv = NS_OK; + + if (mFD) { + if (flush) + rv = FlushBitMap(); + PRStatus err = PR_Close(mFD); + if (NS_SUCCEEDED(rv) && (err != PR_SUCCESS)) + rv = NS_ERROR_UNEXPECTED; + mFD = nullptr; + } + + if (mBitMap) { + delete [] mBitMap; + mBitMap = nullptr; + } + + return rv; +} + + +/****************************************************************************** + * AllocateBlocks + * + * Allocates 1-4 blocks, using a first fit strategy, + * so that no group of blocks spans a quad block boundary. + * + * Returns block number of first block allocated or -1 on failure. + * + *****************************************************************************/ +int32_t +nsDiskCacheBlockFile::AllocateBlocks(int32_t numBlocks) +{ + const int maxPos = 32 - numBlocks; + const uint32_t mask = (0x01 << numBlocks) - 1; + for (unsigned int i = 0; i < mBitMapWords; ++i) { + uint32_t mapWord = ~mBitMap[i]; // flip bits so free bits are 1 + if (mapWord) { // At least one free bit + // Binary search for first free bit in word + int bit = 0; + if ((mapWord & 0x0FFFF) == 0) { bit |= 16; mapWord >>= 16; } + if ((mapWord & 0x000FF) == 0) { bit |= 8; mapWord >>= 8; } + if ((mapWord & 0x0000F) == 0) { bit |= 4; mapWord >>= 4; } + if ((mapWord & 0x00003) == 0) { bit |= 2; mapWord >>= 2; } + if ((mapWord & 0x00001) == 0) { bit |= 1; mapWord >>= 1; } + // Find first fit for mask + for (; bit <= maxPos; ++bit) { + // all bits selected by mask are 1, so free + if ((mask & mapWord) == mask) { + mBitMap[i] |= mask << bit; + mBitMapDirty = true; + return (int32_t)i * 32 + bit; + } + } + } + } + + return -1; +} + + +/****************************************************************************** + * DeallocateBlocks + *****************************************************************************/ +nsresult +nsDiskCacheBlockFile::DeallocateBlocks( int32_t startBlock, int32_t numBlocks) +{ + if (!mFD) return NS_ERROR_NOT_AVAILABLE; + + if ((startBlock < 0) || ((uint32_t)startBlock > mBitMapWords * 32 - 1) || + (numBlocks < 1) || (numBlocks > 4)) + return NS_ERROR_ILLEGAL_VALUE; + + const int32_t startWord = startBlock >> 5; // Divide by 32 + const uint32_t startBit = startBlock & 31; // Modulo by 32 + + // make sure requested deallocation doesn't span a word boundary + if (startBit + numBlocks > 32) return NS_ERROR_UNEXPECTED; + uint32_t mask = ((0x01 << numBlocks) - 1) << startBit; + + // make sure requested deallocation is currently allocated + if ((mBitMap[startWord] & mask) != mask) return NS_ERROR_ABORT; + + mBitMap[startWord] ^= mask; // flips the bits off; + mBitMapDirty = true; + // XXX rv = FlushBitMap(); // coherency vs. performance + return NS_OK; +} + + +/****************************************************************************** + * WriteBlocks + *****************************************************************************/ +nsresult +nsDiskCacheBlockFile::WriteBlocks( void * buffer, + uint32_t size, + int32_t numBlocks, + int32_t * startBlock) +{ + // presume buffer != nullptr and startBlock != nullptr + NS_ENSURE_TRUE(mFD, NS_ERROR_NOT_AVAILABLE); + + // allocate some blocks in the cache block file + *startBlock = AllocateBlocks(numBlocks); + if (*startBlock < 0) + return NS_ERROR_NOT_AVAILABLE; + + // seek to block position + int32_t blockPos = mBitMapWords * 4 + *startBlock * mBlockSize; + + // write the blocks + return Write(blockPos, buffer, size) ? NS_OK : NS_ERROR_FAILURE; +} + + +/****************************************************************************** + * ReadBlocks + *****************************************************************************/ +nsresult +nsDiskCacheBlockFile::ReadBlocks( void * buffer, + int32_t startBlock, + int32_t numBlocks, + int32_t * bytesRead) +{ + // presume buffer != nullptr and bytesRead != bytesRead + + if (!mFD) return NS_ERROR_NOT_AVAILABLE; + nsresult rv = VerifyAllocation(startBlock, numBlocks); + if (NS_FAILED(rv)) return rv; + + // seek to block position + int32_t blockPos = mBitMapWords * 4 + startBlock * mBlockSize; + int32_t filePos = PR_Seek(mFD, blockPos, PR_SEEK_SET); + if (filePos != blockPos) return NS_ERROR_UNEXPECTED; + + // read the blocks + int32_t bytesToRead = *bytesRead; + if ((bytesToRead <= 0) || ((uint32_t)bytesToRead > mBlockSize * numBlocks)) { + bytesToRead = mBlockSize * numBlocks; + } + *bytesRead = PR_Read(mFD, buffer, bytesToRead); + + CACHE_LOG_DEBUG(("CACHE: nsDiskCacheBlockFile::Read [this=%p] " + "returned %d / %d bytes", this, *bytesRead, bytesToRead)); + + return NS_OK; +} + + +/****************************************************************************** + * FlushBitMap + *****************************************************************************/ +nsresult +nsDiskCacheBlockFile::FlushBitMap() +{ + if (!mBitMapDirty) return NS_OK; + +#if defined(IS_LITTLE_ENDIAN) + uint32_t *bitmap = new uint32_t[mBitMapWords]; + // Copy and swap to network format + uint32_t *p = bitmap; + for (unsigned int i = 0; i < mBitMapWords; ++i, ++p) + *p = htonl(mBitMap[i]); +#else + uint32_t *bitmap = mBitMap; +#endif + + // write bitmap + bool written = Write(0, bitmap, mBitMapWords * 4); +#if defined(IS_LITTLE_ENDIAN) + delete [] bitmap; +#endif + if (!written) + return NS_ERROR_UNEXPECTED; + + PRStatus err = PR_Sync(mFD); + if (err != PR_SUCCESS) return NS_ERROR_UNEXPECTED; + + mBitMapDirty = false; + return NS_OK; +} + + +/****************************************************************************** + * VerifyAllocation + * + * Return values: + * NS_OK if all bits are marked allocated + * NS_ERROR_ILLEGAL_VALUE if parameters don't obey constraints + * NS_ERROR_FAILURE if some or all the bits are marked unallocated + * + *****************************************************************************/ +nsresult +nsDiskCacheBlockFile::VerifyAllocation( int32_t startBlock, int32_t numBlocks) +{ + if ((startBlock < 0) || ((uint32_t)startBlock > mBitMapWords * 32 - 1) || + (numBlocks < 1) || (numBlocks > 4)) + return NS_ERROR_ILLEGAL_VALUE; + + const int32_t startWord = startBlock >> 5; // Divide by 32 + const uint32_t startBit = startBlock & 31; // Modulo by 32 + + // make sure requested deallocation doesn't span a word boundary + if (startBit + numBlocks > 32) return NS_ERROR_ILLEGAL_VALUE; + uint32_t mask = ((0x01 << numBlocks) - 1) << startBit; + + // check if all specified blocks are currently allocated + if ((mBitMap[startWord] & mask) != mask) return NS_ERROR_FAILURE; + + return NS_OK; +} + + +/****************************************************************************** + * CalcBlockFileSize + * + * Return size of the block file according to the bits set in mBitmap + * + *****************************************************************************/ +uint32_t +nsDiskCacheBlockFile::CalcBlockFileSize() +{ + // search for last byte in mBitMap with allocated bits + uint32_t estimatedSize = mBitMapWords * 4; + int32_t i = mBitMapWords; + while (--i >= 0) { + if (mBitMap[i]) break; + } + + if (i >= 0) { + // binary search to find last allocated bit in byte + uint32_t mapWord = mBitMap[i]; + uint32_t lastBit = 31; + if ((mapWord & 0xFFFF0000) == 0) { lastBit ^= 16; mapWord <<= 16; } + if ((mapWord & 0xFF000000) == 0) { lastBit ^= 8; mapWord <<= 8; } + if ((mapWord & 0xF0000000) == 0) { lastBit ^= 4; mapWord <<= 4; } + if ((mapWord & 0xC0000000) == 0) { lastBit ^= 2; mapWord <<= 2; } + if ((mapWord & 0x80000000) == 0) { lastBit ^= 1; mapWord <<= 1; } + estimatedSize += (i * 32 + lastBit + 1) * mBlockSize; + } + + return estimatedSize; +} + +/****************************************************************************** + * Write + * + * Wrapper around PR_Write that grows file in larger chunks to combat fragmentation + * + *****************************************************************************/ +bool +nsDiskCacheBlockFile::Write(int32_t offset, const void *buf, int32_t amount) +{ + /* Grow the file to 4mb right away, then double it until the file grows to 20mb. + 20mb is a magic threshold because OSX stops autodefragging files bigger than that. + Beyond 20mb grow in 4mb chunks. + */ + const int32_t upTo = offset + amount; + // Use a conservative definition of 20MB + const int32_t minPreallocate = 4*1024*1024; + const int32_t maxPreallocate = 20*1000*1000; + if (mFileSize < upTo) { + // maximal file size + const int32_t maxFileSize = mBitMapWords * 4 * (mBlockSize * 8 + 1); + if (upTo > maxPreallocate) { + // grow the file as a multiple of minPreallocate + mFileSize = ((upTo + minPreallocate - 1) / minPreallocate) * minPreallocate; + } else { + // Grow quickly between 1MB to 20MB + if (mFileSize) + while(mFileSize < upTo) + mFileSize *= 2; + mFileSize = clamped(mFileSize, minPreallocate, maxPreallocate); + } + mFileSize = std::min(mFileSize, maxFileSize); +#if !defined(XP_MACOSX) + mozilla::fallocate(mFD, mFileSize); +#endif + } + if (PR_Seek(mFD, offset, PR_SEEK_SET) != offset) + return false; + return PR_Write(mFD, buf, amount) == amount; +} + +size_t +nsDiskCacheBlockFile::SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) +{ + return aMallocSizeOf(mBitMap) + aMallocSizeOf(mFD); +} diff --git a/netwerk/cache/nsDiskCacheBlockFile.h b/netwerk/cache/nsDiskCacheBlockFile.h new file mode 100644 index 000000000..058cc7fcc --- /dev/null +++ b/netwerk/cache/nsDiskCacheBlockFile.h @@ -0,0 +1,69 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* vim: set sw=4 ts=8 et tw=80 : */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef _nsDiskCacheBlockFile_h_ +#define _nsDiskCacheBlockFile_h_ + +#include "mozilla/MemoryReporting.h" +#include "nsIFile.h" +#include "nsDiskCache.h" + +/****************************************************************************** + * nsDiskCacheBlockFile + * + * The structure of a cache block file is a 4096 bytes bit map, followed by + * some number of blocks of mBlockSize. The creator of a + * nsDiskCacheBlockFile object must provide the block size for a given file. + * + *****************************************************************************/ +class nsDiskCacheBlockFile { +public: + nsDiskCacheBlockFile() + : mFD(nullptr) + , mBitMap(nullptr) + , mBlockSize(0) + , mBitMapWords(0) + , mFileSize(0) + , mBitMapDirty(false) + {} + ~nsDiskCacheBlockFile() { (void) Close(true); } + + nsresult Open( nsIFile * blockFile, uint32_t blockSize, + uint32_t bitMapSize, nsDiskCache::CorruptCacheInfo * corruptInfo); + nsresult Close(bool flush); + + /* + * Trim + * Truncates the block file to the end of the last allocated block. + */ + nsresult Trim() { return nsDiskCache::Truncate(mFD, CalcBlockFileSize()); } + nsresult DeallocateBlocks( int32_t startBlock, int32_t numBlocks); + nsresult WriteBlocks( void * buffer, uint32_t size, int32_t numBlocks, + int32_t * startBlock); + nsresult ReadBlocks( void * buffer, int32_t startBlock, int32_t numBlocks, + int32_t * bytesRead); + + size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf); + +private: + nsresult FlushBitMap(); + int32_t AllocateBlocks( int32_t numBlocks); + nsresult VerifyAllocation( int32_t startBlock, int32_t numBLocks); + uint32_t CalcBlockFileSize(); + bool Write(int32_t offset, const void *buf, int32_t amount); + +/** + * Data members + */ + PRFileDesc * mFD; + uint32_t * mBitMap; // XXX future: array of bit map blocks + uint32_t mBlockSize; + uint32_t mBitMapWords; + int32_t mFileSize; + bool mBitMapDirty; +}; + +#endif // _nsDiskCacheBlockFile_h_ diff --git a/netwerk/cache/nsDiskCacheDevice.cpp b/netwerk/cache/nsDiskCacheDevice.cpp new file mode 100644 index 000000000..ac91534ff --- /dev/null +++ b/netwerk/cache/nsDiskCacheDevice.cpp @@ -0,0 +1,1149 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include <limits.h> + +#include "mozilla/DebugOnly.h" + +#include "nsCache.h" +#include "nsIMemoryReporter.h" + +// include files for ftruncate (or equivalent) +#if defined(XP_UNIX) +#include <unistd.h> +#elif defined(XP_WIN) +#include <windows.h> +#else +// XXX add necessary include file for ftruncate (or equivalent) +#endif + +#include "prthread.h" + +#include "private/pprio.h" + +#include "nsDiskCacheDevice.h" +#include "nsDiskCacheEntry.h" +#include "nsDiskCacheMap.h" +#include "nsDiskCacheStreams.h" + +#include "nsDiskCache.h" + +#include "nsCacheService.h" + +#include "nsDeleteDir.h" + +#include "nsICacheVisitor.h" +#include "nsReadableUtils.h" +#include "nsIInputStream.h" +#include "nsIOutputStream.h" +#include "nsCRT.h" +#include "nsCOMArray.h" +#include "nsISimpleEnumerator.h" + +#include "nsThreadUtils.h" +#include "mozilla/MemoryReporting.h" +#include "mozilla/Telemetry.h" + +static const char DISK_CACHE_DEVICE_ID[] = { "disk" }; +using namespace mozilla; + +class nsDiskCacheDeviceDeactivateEntryEvent : public Runnable { +public: + nsDiskCacheDeviceDeactivateEntryEvent(nsDiskCacheDevice *device, + nsCacheEntry * entry, + nsDiskCacheBinding * binding) + : mCanceled(false), + mEntry(entry), + mDevice(device), + mBinding(binding) + { + } + + NS_IMETHOD Run() override + { + nsCacheServiceAutoLock lock; + CACHE_LOG_DEBUG(("nsDiskCacheDeviceDeactivateEntryEvent[%p]\n", this)); + if (!mCanceled) { + (void) mDevice->DeactivateEntry_Private(mEntry, mBinding); + } + return NS_OK; + } + + void CancelEvent() { mCanceled = true; } +private: + bool mCanceled; + nsCacheEntry *mEntry; + nsDiskCacheDevice *mDevice; + nsDiskCacheBinding *mBinding; +}; + +class nsEvictDiskCacheEntriesEvent : public Runnable { +public: + explicit nsEvictDiskCacheEntriesEvent(nsDiskCacheDevice *device) + : mDevice(device) {} + + NS_IMETHOD Run() override + { + nsCacheServiceAutoLock lock; + mDevice->EvictDiskCacheEntries(mDevice->mCacheCapacity); + return NS_OK; + } + +private: + nsDiskCacheDevice *mDevice; +}; + +/****************************************************************************** + * nsDiskCacheEvictor + * + * Helper class for nsDiskCacheDevice. + * + *****************************************************************************/ + +class nsDiskCacheEvictor : public nsDiskCacheRecordVisitor +{ +public: + nsDiskCacheEvictor( nsDiskCacheMap * cacheMap, + nsDiskCacheBindery * cacheBindery, + uint32_t targetSize, + const char * clientID) + : mCacheMap(cacheMap) + , mBindery(cacheBindery) + , mTargetSize(targetSize) + , mClientID(clientID) + { + mClientIDSize = clientID ? strlen(clientID) : 0; + } + + virtual int32_t VisitRecord(nsDiskCacheRecord * mapRecord); + +private: + nsDiskCacheMap * mCacheMap; + nsDiskCacheBindery * mBindery; + uint32_t mTargetSize; + const char * mClientID; + uint32_t mClientIDSize; +}; + + +int32_t +nsDiskCacheEvictor::VisitRecord(nsDiskCacheRecord * mapRecord) +{ + if (mCacheMap->TotalSize() < mTargetSize) + return kStopVisitingRecords; + + if (mClientID) { + // we're just evicting records for a specific client + nsDiskCacheEntry * diskEntry = mCacheMap->ReadDiskCacheEntry(mapRecord); + if (!diskEntry) + return kVisitNextRecord; // XXX or delete record? + + // Compare clientID's without malloc + if ((diskEntry->mKeySize <= mClientIDSize) || + (diskEntry->Key()[mClientIDSize] != ':') || + (memcmp(diskEntry->Key(), mClientID, mClientIDSize) != 0)) { + return kVisitNextRecord; // clientID doesn't match, skip it + } + } + + nsDiskCacheBinding * binding = mBindery->FindActiveBinding(mapRecord->HashNumber()); + if (binding) { + // If the entry is pending deactivation, cancel deactivation and doom + // the entry + if (binding->mDeactivateEvent) { + binding->mDeactivateEvent->CancelEvent(); + binding->mDeactivateEvent = nullptr; + } + // We are currently using this entry, so all we can do is doom it. + // Since we're enumerating the records, we don't want to call + // DeleteRecord when nsCacheService::DoomEntry() calls us back. + binding->mDoomed = true; // mark binding record as 'deleted' + nsCacheService::DoomEntry(binding->mCacheEntry); + } else { + // entry not in use, just delete storage because we're enumerating the records + (void) mCacheMap->DeleteStorage(mapRecord); + } + + return kDeleteRecordAndContinue; // this will REALLY delete the record +} + + +/****************************************************************************** + * nsDiskCacheDeviceInfo + *****************************************************************************/ + +class nsDiskCacheDeviceInfo : public nsICacheDeviceInfo { +public: + NS_DECL_ISUPPORTS + NS_DECL_NSICACHEDEVICEINFO + + explicit nsDiskCacheDeviceInfo(nsDiskCacheDevice* device) + : mDevice(device) + { + } + +private: + virtual ~nsDiskCacheDeviceInfo() {} + + nsDiskCacheDevice* mDevice; +}; + +NS_IMPL_ISUPPORTS(nsDiskCacheDeviceInfo, nsICacheDeviceInfo) + +NS_IMETHODIMP nsDiskCacheDeviceInfo::GetDescription(char ** aDescription) +{ + NS_ENSURE_ARG_POINTER(aDescription); + *aDescription = NS_strdup("Disk cache device"); + return *aDescription ? NS_OK : NS_ERROR_OUT_OF_MEMORY; +} + +NS_IMETHODIMP nsDiskCacheDeviceInfo::GetUsageReport(char ** usageReport) +{ + NS_ENSURE_ARG_POINTER(usageReport); + nsCString buffer; + + buffer.AssignLiteral(" <tr>\n" + " <th>Cache Directory:</th>\n" + " <td>"); + nsCOMPtr<nsIFile> cacheDir; + nsAutoString path; + mDevice->getCacheDirectory(getter_AddRefs(cacheDir)); + nsresult rv = cacheDir->GetPath(path); + if (NS_SUCCEEDED(rv)) { + AppendUTF16toUTF8(path, buffer); + } else { + buffer.AppendLiteral("directory unavailable"); + } + buffer.AppendLiteral("</td>\n" + " </tr>\n"); + + *usageReport = ToNewCString(buffer); + if (!*usageReport) return NS_ERROR_OUT_OF_MEMORY; + + return NS_OK; +} + +NS_IMETHODIMP nsDiskCacheDeviceInfo::GetEntryCount(uint32_t *aEntryCount) +{ + NS_ENSURE_ARG_POINTER(aEntryCount); + *aEntryCount = mDevice->getEntryCount(); + return NS_OK; +} + +NS_IMETHODIMP nsDiskCacheDeviceInfo::GetTotalSize(uint32_t *aTotalSize) +{ + NS_ENSURE_ARG_POINTER(aTotalSize); + // Returned unit's are in bytes + *aTotalSize = mDevice->getCacheSize() * 1024; + return NS_OK; +} + +NS_IMETHODIMP nsDiskCacheDeviceInfo::GetMaximumSize(uint32_t *aMaximumSize) +{ + NS_ENSURE_ARG_POINTER(aMaximumSize); + // Returned unit's are in bytes + *aMaximumSize = mDevice->getCacheCapacity() * 1024; + return NS_OK; +} + + +/****************************************************************************** + * nsDiskCache + *****************************************************************************/ + +/** + * nsDiskCache::Hash(const char * key, PLDHashNumber initval) + * + * See http://burtleburtle.net/bob/hash/evahash.html for more information + * about this hash function. + * + * This algorithm of this method implies nsDiskCacheRecords will be stored + * in a certain order on disk. If the algorithm changes, existing cache + * map files may become invalid, and therefore the kCurrentVersion needs + * to be revised. + */ + +static inline void hashmix(uint32_t& a, uint32_t& b, uint32_t& c) +{ + a -= b; a -= c; a ^= (c>>13); + b -= c; b -= a; b ^= (a<<8); + c -= a; c -= b; c ^= (b>>13); + a -= b; a -= c; a ^= (c>>12); + b -= c; b -= a; b ^= (a<<16); + c -= a; c -= b; c ^= (b>>5); + a -= b; a -= c; a ^= (c>>3); + b -= c; b -= a; b ^= (a<<10); + c -= a; c -= b; c ^= (b>>15); +} + +PLDHashNumber +nsDiskCache::Hash(const char * key, PLDHashNumber initval) +{ + const uint8_t *k = reinterpret_cast<const uint8_t*>(key); + uint32_t a, b, c, len, length; + + length = strlen(key); + /* Set up the internal state */ + len = length; + a = b = 0x9e3779b9; /* the golden ratio; an arbitrary value */ + c = initval; /* variable initialization of internal state */ + + /*---------------------------------------- handle most of the key */ + while (len >= 12) + { + a += k[0] + (uint32_t(k[1])<<8) + (uint32_t(k[2])<<16) + (uint32_t(k[3])<<24); + b += k[4] + (uint32_t(k[5])<<8) + (uint32_t(k[6])<<16) + (uint32_t(k[7])<<24); + c += k[8] + (uint32_t(k[9])<<8) + (uint32_t(k[10])<<16) + (uint32_t(k[11])<<24); + hashmix(a, b, c); + k += 12; len -= 12; + } + + /*------------------------------------- handle the last 11 bytes */ + c += length; + switch(len) { /* all the case statements fall through */ + case 11: c += (uint32_t(k[10])<<24); MOZ_FALLTHROUGH; + case 10: c += (uint32_t(k[9])<<16); MOZ_FALLTHROUGH; + case 9 : c += (uint32_t(k[8])<<8); MOZ_FALLTHROUGH; + /* the low-order byte of c is reserved for the length */ + case 8 : b += (uint32_t(k[7])<<24); MOZ_FALLTHROUGH; + case 7 : b += (uint32_t(k[6])<<16); MOZ_FALLTHROUGH; + case 6 : b += (uint32_t(k[5])<<8); MOZ_FALLTHROUGH; + case 5 : b += k[4]; MOZ_FALLTHROUGH; + case 4 : a += (uint32_t(k[3])<<24); MOZ_FALLTHROUGH; + case 3 : a += (uint32_t(k[2])<<16); MOZ_FALLTHROUGH; + case 2 : a += (uint32_t(k[1])<<8); MOZ_FALLTHROUGH; + case 1 : a += k[0]; + /* case 0: nothing left to add */ + } + hashmix(a, b, c); + + return c; +} + +nsresult +nsDiskCache::Truncate(PRFileDesc * fd, uint32_t newEOF) +{ + // use modified SetEOF from nsFileStreams::SetEOF() + +#if defined(XP_UNIX) + if (ftruncate(PR_FileDesc2NativeHandle(fd), newEOF) != 0) { + NS_ERROR("ftruncate failed"); + return NS_ERROR_FAILURE; + } + +#elif defined(XP_WIN) + int32_t cnt = PR_Seek(fd, newEOF, PR_SEEK_SET); + if (cnt == -1) return NS_ERROR_FAILURE; + if (!SetEndOfFile((HANDLE) PR_FileDesc2NativeHandle(fd))) { + NS_ERROR("SetEndOfFile failed"); + return NS_ERROR_FAILURE; + } + +#else + // add implementations for other platforms here +#endif + return NS_OK; +} + + +/****************************************************************************** + * nsDiskCacheDevice + *****************************************************************************/ + +nsDiskCacheDevice::nsDiskCacheDevice() + : mCacheCapacity(0) + , mMaxEntrySize(-1) // -1 means "no limit" + , mInitialized(false) + , mClearingDiskCache(false) +{ +} + +nsDiskCacheDevice::~nsDiskCacheDevice() +{ + Shutdown(); +} + + +/** + * methods of nsCacheDevice + */ +nsresult +nsDiskCacheDevice::Init() +{ + nsresult rv; + + if (Initialized()) { + NS_ERROR("Disk cache already initialized!"); + return NS_ERROR_UNEXPECTED; + } + + if (!mCacheDirectory) + return NS_ERROR_FAILURE; + + mBindery.Init(); + + // Open Disk Cache + rv = OpenDiskCache(); + if (NS_FAILED(rv)) { + (void) mCacheMap.Close(false); + return rv; + } + + mInitialized = true; + return NS_OK; +} + + +/** + * NOTE: called while holding the cache service lock + */ +nsresult +nsDiskCacheDevice::Shutdown() +{ + nsCacheService::AssertOwnsLock(); + + nsresult rv = Shutdown_Private(true); + if (NS_FAILED(rv)) + return rv; + + return NS_OK; +} + + +nsresult +nsDiskCacheDevice::Shutdown_Private(bool flush) +{ + CACHE_LOG_DEBUG(("CACHE: disk Shutdown_Private [%u]\n", flush)); + + if (Initialized()) { + // check cache limits in case we need to evict. + EvictDiskCacheEntries(mCacheCapacity); + + // At this point there may be a number of pending cache-requests on the + // cache-io thread. Wait for all these to run before we wipe out our + // datastructures (see bug #620660) + (void) nsCacheService::SyncWithCacheIOThread(); + + // write out persistent information about the cache. + (void) mCacheMap.Close(flush); + + mBindery.Reset(); + + mInitialized = false; + } + + return NS_OK; +} + + +const char * +nsDiskCacheDevice::GetDeviceID() +{ + return DISK_CACHE_DEVICE_ID; +} + +/** + * FindEntry - + * + * cases: key not in disk cache, hash number free + * key not in disk cache, hash number used + * key in disk cache + * + * NOTE: called while holding the cache service lock + */ +nsCacheEntry * +nsDiskCacheDevice::FindEntry(nsCString * key, bool *collision) +{ + Telemetry::AutoTimer<Telemetry::CACHE_DISK_SEARCH_2> timer; + if (!Initialized()) return nullptr; // NS_ERROR_NOT_INITIALIZED + if (mClearingDiskCache) return nullptr; + nsDiskCacheRecord record; + nsDiskCacheBinding * binding = nullptr; + PLDHashNumber hashNumber = nsDiskCache::Hash(key->get()); + + *collision = false; + + binding = mBindery.FindActiveBinding(hashNumber); + if (binding && !binding->mCacheEntry->Key()->Equals(*key)) { + *collision = true; + return nullptr; + } else if (binding && binding->mDeactivateEvent) { + binding->mDeactivateEvent->CancelEvent(); + binding->mDeactivateEvent = nullptr; + CACHE_LOG_DEBUG(("CACHE: reusing deactivated entry %p " \ + "req-key=%s entry-key=%s\n", + binding->mCacheEntry, key, binding->mCacheEntry->Key())); + + return binding->mCacheEntry; // just return this one, observing that + // FindActiveBinding() does not return + // bindings to doomed entries + } + binding = nullptr; + + // lookup hash number in cache map + nsresult rv = mCacheMap.FindRecord(hashNumber, &record); + if (NS_FAILED(rv)) return nullptr; // XXX log error? + + nsDiskCacheEntry * diskEntry = mCacheMap.ReadDiskCacheEntry(&record); + if (!diskEntry) return nullptr; + + // compare key to be sure + if (!key->Equals(diskEntry->Key())) { + *collision = true; + return nullptr; + } + + nsCacheEntry * entry = diskEntry->CreateCacheEntry(this); + if (entry) { + binding = mBindery.CreateBinding(entry, &record); + if (!binding) { + delete entry; + entry = nullptr; + } + } + + if (!entry) { + (void) mCacheMap.DeleteStorage(&record); + (void) mCacheMap.DeleteRecord(&record); + } + + return entry; +} + + +/** + * NOTE: called while holding the cache service lock + */ +nsresult +nsDiskCacheDevice::DeactivateEntry(nsCacheEntry * entry) +{ + nsDiskCacheBinding * binding = GetCacheEntryBinding(entry); + if (!IsValidBinding(binding)) + return NS_ERROR_UNEXPECTED; + + CACHE_LOG_DEBUG(("CACHE: disk DeactivateEntry [%p %x]\n", + entry, binding->mRecord.HashNumber())); + + nsDiskCacheDeviceDeactivateEntryEvent *event = + new nsDiskCacheDeviceDeactivateEntryEvent(this, entry, binding); + + // ensure we can cancel the event via the binding later if necessary + binding->mDeactivateEvent = event; + + DebugOnly<nsresult> rv = nsCacheService::DispatchToCacheIOThread(event); + NS_ASSERTION(NS_SUCCEEDED(rv), "DeactivateEntry: Failed dispatching " + "deactivation event"); + return NS_OK; +} + +/** + * NOTE: called while holding the cache service lock + */ +nsresult +nsDiskCacheDevice::DeactivateEntry_Private(nsCacheEntry * entry, + nsDiskCacheBinding * binding) +{ + nsresult rv = NS_OK; + if (entry->IsDoomed()) { + // delete data, entry, record from disk for entry + rv = mCacheMap.DeleteStorage(&binding->mRecord); + + } else { + // save stuff to disk for entry + rv = mCacheMap.WriteDiskCacheEntry(binding); + if (NS_FAILED(rv)) { + // clean up as best we can + (void) mCacheMap.DeleteStorage(&binding->mRecord); + (void) mCacheMap.DeleteRecord(&binding->mRecord); + binding->mDoomed = true; // record is no longer in cache map + } + } + + mBindery.RemoveBinding(binding); // extract binding from collision detection stuff + delete entry; // which will release binding + return rv; +} + + +/** + * BindEntry() + * no hash number collision -> no problem + * collision + * record not active -> evict, no problem + * record is active + * record is already doomed -> record shouldn't have been in map, no problem + * record is not doomed -> doom, and replace record in map + * + * walk matching hashnumber list to find lowest generation number + * take generation number from other (data/meta) location, + * or walk active list + * + * NOTE: called while holding the cache service lock + */ +nsresult +nsDiskCacheDevice::BindEntry(nsCacheEntry * entry) +{ + if (!Initialized()) return NS_ERROR_NOT_INITIALIZED; + if (mClearingDiskCache) return NS_ERROR_NOT_AVAILABLE; + nsresult rv = NS_OK; + nsDiskCacheRecord record, oldRecord; + nsDiskCacheBinding *binding; + PLDHashNumber hashNumber = nsDiskCache::Hash(entry->Key()->get()); + + // Find out if there is already an active binding for this hash. If yes it + // should have another key since BindEntry() shouldn't be called twice for + // the same entry. Doom the old entry, the new one will get another + // generation number so files won't collide. + binding = mBindery.FindActiveBinding(hashNumber); + if (binding) { + NS_ASSERTION(!binding->mCacheEntry->Key()->Equals(*entry->Key()), + "BindEntry called for already bound entry!"); + // If the entry is pending deactivation, cancel deactivation + if (binding->mDeactivateEvent) { + binding->mDeactivateEvent->CancelEvent(); + binding->mDeactivateEvent = nullptr; + } + nsCacheService::DoomEntry(binding->mCacheEntry); + binding = nullptr; + } + + // Lookup hash number in cache map. There can be a colliding inactive entry. + // See bug #321361 comment 21 for the scenario. If there is such entry, + // delete it. + rv = mCacheMap.FindRecord(hashNumber, &record); + if (NS_SUCCEEDED(rv)) { + nsDiskCacheEntry * diskEntry = mCacheMap.ReadDiskCacheEntry(&record); + if (diskEntry) { + // compare key to be sure + if (!entry->Key()->Equals(diskEntry->Key())) { + mCacheMap.DeleteStorage(&record); + rv = mCacheMap.DeleteRecord(&record); + if (NS_FAILED(rv)) return rv; + } + } + record = nsDiskCacheRecord(); + } + + // create a new record for this entry + record.SetHashNumber(nsDiskCache::Hash(entry->Key()->get())); + record.SetEvictionRank(ULONG_MAX - SecondsFromPRTime(PR_Now())); + + CACHE_LOG_DEBUG(("CACHE: disk BindEntry [%p %x]\n", + entry, record.HashNumber())); + + if (!entry->IsDoomed()) { + // if entry isn't doomed, add it to the cache map + rv = mCacheMap.AddRecord(&record, &oldRecord); // deletes old record, if any + if (NS_FAILED(rv)) return rv; + + uint32_t oldHashNumber = oldRecord.HashNumber(); + if (oldHashNumber) { + // gotta evict this one first + nsDiskCacheBinding * oldBinding = mBindery.FindActiveBinding(oldHashNumber); + if (oldBinding) { + // XXX if debug : compare keys for hashNumber collision + + if (!oldBinding->mCacheEntry->IsDoomed()) { + // If the old entry is pending deactivation, cancel deactivation + if (oldBinding->mDeactivateEvent) { + oldBinding->mDeactivateEvent->CancelEvent(); + oldBinding->mDeactivateEvent = nullptr; + } + // we've got a live one! + nsCacheService::DoomEntry(oldBinding->mCacheEntry); + // storage will be delete when oldBinding->mCacheEntry is Deactivated + } + } else { + // delete storage + // XXX if debug : compare keys for hashNumber collision + rv = mCacheMap.DeleteStorage(&oldRecord); + if (NS_FAILED(rv)) return rv; // XXX delete record we just added? + } + } + } + + // Make sure this entry has its associated nsDiskCacheBinding attached. + binding = mBindery.CreateBinding(entry, &record); + NS_ASSERTION(binding, "nsDiskCacheDevice::BindEntry"); + if (!binding) return NS_ERROR_OUT_OF_MEMORY; + NS_ASSERTION(binding->mRecord.ValidRecord(), "bad cache map record"); + + return NS_OK; +} + + +/** + * NOTE: called while holding the cache service lock + */ +void +nsDiskCacheDevice::DoomEntry(nsCacheEntry * entry) +{ + CACHE_LOG_DEBUG(("CACHE: disk DoomEntry [%p]\n", entry)); + + nsDiskCacheBinding * binding = GetCacheEntryBinding(entry); + NS_ASSERTION(binding, "DoomEntry: binding == nullptr"); + if (!binding) + return; + + if (!binding->mDoomed) { + // so it can't be seen by FindEntry() ever again. +#ifdef DEBUG + nsresult rv = +#endif + mCacheMap.DeleteRecord(&binding->mRecord); + NS_ASSERTION(NS_SUCCEEDED(rv),"DeleteRecord failed."); + binding->mDoomed = true; // record in no longer in cache map + } +} + + +/** + * NOTE: called while holding the cache service lock + */ +nsresult +nsDiskCacheDevice::OpenInputStreamForEntry(nsCacheEntry * entry, + nsCacheAccessMode mode, + uint32_t offset, + nsIInputStream ** result) +{ + CACHE_LOG_DEBUG(("CACHE: disk OpenInputStreamForEntry [%p %x %u]\n", + entry, mode, offset)); + + NS_ENSURE_ARG_POINTER(entry); + NS_ENSURE_ARG_POINTER(result); + + nsresult rv; + nsDiskCacheBinding * binding = GetCacheEntryBinding(entry); + if (!IsValidBinding(binding)) + return NS_ERROR_UNEXPECTED; + + NS_ASSERTION(binding->mCacheEntry == entry, "binding & entry don't point to each other"); + + rv = binding->EnsureStreamIO(); + if (NS_FAILED(rv)) return rv; + + return binding->mStreamIO->GetInputStream(offset, result); +} + + +/** + * NOTE: called while holding the cache service lock + */ +nsresult +nsDiskCacheDevice::OpenOutputStreamForEntry(nsCacheEntry * entry, + nsCacheAccessMode mode, + uint32_t offset, + nsIOutputStream ** result) +{ + CACHE_LOG_DEBUG(("CACHE: disk OpenOutputStreamForEntry [%p %x %u]\n", + entry, mode, offset)); + + NS_ENSURE_ARG_POINTER(entry); + NS_ENSURE_ARG_POINTER(result); + + nsresult rv; + nsDiskCacheBinding * binding = GetCacheEntryBinding(entry); + if (!IsValidBinding(binding)) + return NS_ERROR_UNEXPECTED; + + NS_ASSERTION(binding->mCacheEntry == entry, "binding & entry don't point to each other"); + + rv = binding->EnsureStreamIO(); + if (NS_FAILED(rv)) return rv; + + return binding->mStreamIO->GetOutputStream(offset, result); +} + + +/** + * NOTE: called while holding the cache service lock + */ +nsresult +nsDiskCacheDevice::GetFileForEntry(nsCacheEntry * entry, + nsIFile ** result) +{ + NS_ENSURE_ARG_POINTER(result); + *result = nullptr; + + nsresult rv; + + nsDiskCacheBinding * binding = GetCacheEntryBinding(entry); + if (!IsValidBinding(binding)) + return NS_ERROR_UNEXPECTED; + + // check/set binding->mRecord for separate file, sync w/mCacheMap + if (binding->mRecord.DataLocationInitialized()) { + if (binding->mRecord.DataFile() != 0) + return NS_ERROR_NOT_AVAILABLE; // data not stored as separate file + + NS_ASSERTION(binding->mRecord.DataFileGeneration() == binding->mGeneration, "error generations out of sync"); + } else { + binding->mRecord.SetDataFileGeneration(binding->mGeneration); + binding->mRecord.SetDataFileSize(0); // 1k minimum + if (!binding->mDoomed) { + // record stored in cache map, so update it + rv = mCacheMap.UpdateRecord(&binding->mRecord); + if (NS_FAILED(rv)) return rv; + } + } + + nsCOMPtr<nsIFile> file; + rv = mCacheMap.GetFileForDiskCacheRecord(&binding->mRecord, + nsDiskCache::kData, + false, + getter_AddRefs(file)); + if (NS_FAILED(rv)) return rv; + + NS_IF_ADDREF(*result = file); + return NS_OK; +} + + +/** + * This routine will get called every time an open descriptor is written to. + * + * NOTE: called while holding the cache service lock + */ +nsresult +nsDiskCacheDevice::OnDataSizeChange(nsCacheEntry * entry, int32_t deltaSize) +{ + CACHE_LOG_DEBUG(("CACHE: disk OnDataSizeChange [%p %d]\n", + entry, deltaSize)); + + // If passed a negative value, then there's nothing to do. + if (deltaSize < 0) + return NS_OK; + + nsDiskCacheBinding * binding = GetCacheEntryBinding(entry); + if (!IsValidBinding(binding)) + return NS_ERROR_UNEXPECTED; + + NS_ASSERTION(binding->mRecord.ValidRecord(), "bad record"); + + uint32_t newSize = entry->DataSize() + deltaSize; + uint32_t newSizeK = ((newSize + 0x3FF) >> 10); + + // If the new size is larger than max. file size or larger than + // 1/8 the cache capacity (which is in KiB's), doom the entry and abort. + if (EntryIsTooBig(newSize)) { +#ifdef DEBUG + nsresult rv = +#endif + nsCacheService::DoomEntry(entry); + NS_ASSERTION(NS_SUCCEEDED(rv),"DoomEntry() failed."); + return NS_ERROR_ABORT; + } + + uint32_t sizeK = ((entry->DataSize() + 0x03FF) >> 10); // round up to next 1k + + // In total count we ignore anything over kMaxDataSizeK (bug #651100), so + // the target capacity should be calculated the same way. + if (sizeK > kMaxDataSizeK) sizeK = kMaxDataSizeK; + if (newSizeK > kMaxDataSizeK) newSizeK = kMaxDataSizeK; + + // pre-evict entries to make space for new data + uint32_t targetCapacity = mCacheCapacity > (newSizeK - sizeK) + ? mCacheCapacity - (newSizeK - sizeK) + : 0; + EvictDiskCacheEntries(targetCapacity); + + return NS_OK; +} + + +/****************************************************************************** + * EntryInfoVisitor + *****************************************************************************/ +class EntryInfoVisitor : public nsDiskCacheRecordVisitor +{ +public: + EntryInfoVisitor(nsDiskCacheMap * cacheMap, + nsICacheVisitor * visitor) + : mCacheMap(cacheMap) + , mVisitor(visitor) + {} + + virtual int32_t VisitRecord(nsDiskCacheRecord * mapRecord) + { + // XXX optimization: do we have this record in memory? + + // read in the entry (metadata) + nsDiskCacheEntry * diskEntry = mCacheMap->ReadDiskCacheEntry(mapRecord); + if (!diskEntry) { + return kVisitNextRecord; + } + + // create nsICacheEntryInfo + nsDiskCacheEntryInfo * entryInfo = new nsDiskCacheEntryInfo(DISK_CACHE_DEVICE_ID, diskEntry); + if (!entryInfo) { + return kStopVisitingRecords; + } + nsCOMPtr<nsICacheEntryInfo> ref(entryInfo); + + bool keepGoing; + (void)mVisitor->VisitEntry(DISK_CACHE_DEVICE_ID, entryInfo, &keepGoing); + return keepGoing ? kVisitNextRecord : kStopVisitingRecords; + } + +private: + nsDiskCacheMap * mCacheMap; + nsICacheVisitor * mVisitor; +}; + + +nsresult +nsDiskCacheDevice::Visit(nsICacheVisitor * visitor) +{ + if (!Initialized()) return NS_ERROR_NOT_INITIALIZED; + nsDiskCacheDeviceInfo* deviceInfo = new nsDiskCacheDeviceInfo(this); + nsCOMPtr<nsICacheDeviceInfo> ref(deviceInfo); + + bool keepGoing; + nsresult rv = visitor->VisitDevice(DISK_CACHE_DEVICE_ID, deviceInfo, &keepGoing); + if (NS_FAILED(rv)) return rv; + + if (keepGoing) { + EntryInfoVisitor infoVisitor(&mCacheMap, visitor); + return mCacheMap.VisitRecords(&infoVisitor); + } + + return NS_OK; +} + +// Max allowed size for an entry is currently MIN(mMaxEntrySize, 1/8 CacheCapacity) +bool +nsDiskCacheDevice::EntryIsTooBig(int64_t entrySize) +{ + if (mMaxEntrySize == -1) // no limit + return entrySize > (static_cast<int64_t>(mCacheCapacity) * 1024 / 8); + else + return entrySize > mMaxEntrySize || + entrySize > (static_cast<int64_t>(mCacheCapacity) * 1024 / 8); +} + +nsresult +nsDiskCacheDevice::EvictEntries(const char * clientID) +{ + CACHE_LOG_DEBUG(("CACHE: disk EvictEntries [%s]\n", clientID)); + + if (!Initialized()) return NS_ERROR_NOT_INITIALIZED; + nsresult rv; + + if (clientID == nullptr) { + // we're clearing the entire disk cache + rv = ClearDiskCache(); + if (rv != NS_ERROR_CACHE_IN_USE) + return rv; + } + + nsDiskCacheEvictor evictor(&mCacheMap, &mBindery, 0, clientID); + rv = mCacheMap.VisitRecords(&evictor); + + if (clientID == nullptr) // we tried to clear the entire cache + rv = mCacheMap.Trim(); // so trim cache block files (if possible) + return rv; +} + + +/** + * private methods + */ + +nsresult +nsDiskCacheDevice::OpenDiskCache() +{ + Telemetry::AutoTimer<Telemetry::NETWORK_DISK_CACHE_OPEN> timer; + // if we don't have a cache directory, create one and open it + bool exists; + nsresult rv = mCacheDirectory->Exists(&exists); + if (NS_FAILED(rv)) + return rv; + + if (exists) { + // Try opening cache map file. + nsDiskCache::CorruptCacheInfo corruptInfo; + rv = mCacheMap.Open(mCacheDirectory, &corruptInfo); + + if (rv == NS_ERROR_ALREADY_INITIALIZED) { + NS_WARNING("nsDiskCacheDevice::OpenDiskCache: already open!"); + } else if (NS_FAILED(rv)) { + // Consider cache corrupt: delete it + // delay delete by 1 minute to avoid IO thrash at startup + rv = nsDeleteDir::DeleteDir(mCacheDirectory, true, 60000); + if (NS_FAILED(rv)) + return rv; + exists = false; + } + } + + // if we don't have a cache directory, create one and open it + if (!exists) { + nsCacheService::MarkStartingFresh(); + rv = mCacheDirectory->Create(nsIFile::DIRECTORY_TYPE, 0777); + CACHE_LOG_PATH(LogLevel::Info, "\ncreate cache directory: %s\n", mCacheDirectory); + CACHE_LOG_INFO(("mCacheDirectory->Create() = %x\n", rv)); + if (NS_FAILED(rv)) + return rv; + + // reopen the cache map + nsDiskCache::CorruptCacheInfo corruptInfo; + rv = mCacheMap.Open(mCacheDirectory, &corruptInfo); + if (NS_FAILED(rv)) + return rv; + } + + return NS_OK; +} + + +nsresult +nsDiskCacheDevice::ClearDiskCache() +{ + if (mBindery.ActiveBindings()) + return NS_ERROR_CACHE_IN_USE; + + mClearingDiskCache = true; + + nsresult rv = Shutdown_Private(false); // false: don't bother flushing + if (NS_FAILED(rv)) + return rv; + + mClearingDiskCache = false; + + // If the disk cache directory is already gone, then it's not an error if + // we fail to delete it ;-) + rv = nsDeleteDir::DeleteDir(mCacheDirectory, true); + if (NS_FAILED(rv) && rv != NS_ERROR_FILE_TARGET_DOES_NOT_EXIST) + return rv; + + return Init(); +} + + +nsresult +nsDiskCacheDevice::EvictDiskCacheEntries(uint32_t targetCapacity) +{ + CACHE_LOG_DEBUG(("CACHE: disk EvictDiskCacheEntries [%u]\n", + targetCapacity)); + + NS_ASSERTION(targetCapacity > 0, "oops"); + + if (mCacheMap.TotalSize() < targetCapacity) + return NS_OK; + + // targetCapacity is in KiB's + nsDiskCacheEvictor evictor(&mCacheMap, &mBindery, targetCapacity, nullptr); + return mCacheMap.EvictRecords(&evictor); +} + + +/** + * methods for prefs + */ + +void +nsDiskCacheDevice::SetCacheParentDirectory(nsIFile * parentDir) +{ + nsresult rv; + bool exists; + + if (Initialized()) { + NS_ASSERTION(false, "Cannot switch cache directory when initialized"); + return; + } + + if (!parentDir) { + mCacheDirectory = nullptr; + return; + } + + // ensure parent directory exists + rv = parentDir->Exists(&exists); + if (NS_SUCCEEDED(rv) && !exists) + rv = parentDir->Create(nsIFile::DIRECTORY_TYPE, 0700); + if (NS_FAILED(rv)) return; + + // ensure cache directory exists + nsCOMPtr<nsIFile> directory; + + rv = parentDir->Clone(getter_AddRefs(directory)); + if (NS_FAILED(rv)) return; + rv = directory->AppendNative(NS_LITERAL_CSTRING("Cache")); + if (NS_FAILED(rv)) return; + + mCacheDirectory = do_QueryInterface(directory); +} + + +void +nsDiskCacheDevice::getCacheDirectory(nsIFile ** result) +{ + *result = mCacheDirectory; + NS_IF_ADDREF(*result); +} + + +/** + * NOTE: called while holding the cache service lock + */ +void +nsDiskCacheDevice::SetCapacity(uint32_t capacity) +{ + // Units are KiB's + mCacheCapacity = capacity; + if (Initialized()) { + if (NS_IsMainThread()) { + // Do not evict entries on the main thread + nsCacheService::DispatchToCacheIOThread( + new nsEvictDiskCacheEntriesEvent(this)); + } else { + // start evicting entries if the new size is smaller! + EvictDiskCacheEntries(mCacheCapacity); + } + } + // Let cache map know of the new capacity + mCacheMap.NotifyCapacityChange(capacity); +} + + +uint32_t nsDiskCacheDevice::getCacheCapacity() +{ + return mCacheCapacity; +} + + +uint32_t nsDiskCacheDevice::getCacheSize() +{ + return mCacheMap.TotalSize(); +} + + +uint32_t nsDiskCacheDevice::getEntryCount() +{ + return mCacheMap.EntryCount(); +} + +void +nsDiskCacheDevice::SetMaxEntrySize(int32_t maxSizeInKilobytes) +{ + // Internal units are bytes. Changing this only takes effect *after* the + // change and has no consequences for existing cache-entries + if (maxSizeInKilobytes >= 0) + mMaxEntrySize = maxSizeInKilobytes * 1024; + else + mMaxEntrySize = -1; +} + +size_t +nsDiskCacheDevice::SizeOfIncludingThis(MallocSizeOf aMallocSizeOf) +{ + size_t usage = aMallocSizeOf(this); + + usage += mCacheMap.SizeOfExcludingThis(aMallocSizeOf); + usage += mBindery.SizeOfExcludingThis(aMallocSizeOf); + + return usage; +} diff --git a/netwerk/cache/nsDiskCacheDevice.h b/netwerk/cache/nsDiskCacheDevice.h new file mode 100644 index 000000000..f986154e6 --- /dev/null +++ b/netwerk/cache/nsDiskCacheDevice.h @@ -0,0 +1,116 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef _nsDiskCacheDevice_h_ +#define _nsDiskCacheDevice_h_ + +#include "mozilla/MemoryReporting.h" +#include "nsCacheDevice.h" +#include "nsDiskCacheBinding.h" +#include "nsDiskCacheBlockFile.h" +#include "nsDiskCacheEntry.h" + +#include "nsIFile.h" +#include "nsIObserver.h" +#include "nsCOMArray.h" + +class nsDiskCacheMap; + + +class nsDiskCacheDevice final : public nsCacheDevice { +public: + nsDiskCacheDevice(); + virtual ~nsDiskCacheDevice(); + + virtual nsresult Init(); + virtual nsresult Shutdown(); + + virtual const char * GetDeviceID(void); + virtual nsCacheEntry * FindEntry(nsCString * key, bool *collision); + virtual nsresult DeactivateEntry(nsCacheEntry * entry); + virtual nsresult BindEntry(nsCacheEntry * entry); + virtual void DoomEntry( nsCacheEntry * entry ); + + virtual nsresult OpenInputStreamForEntry(nsCacheEntry * entry, + nsCacheAccessMode mode, + uint32_t offset, + nsIInputStream ** result); + + virtual nsresult OpenOutputStreamForEntry(nsCacheEntry * entry, + nsCacheAccessMode mode, + uint32_t offset, + nsIOutputStream ** result); + + virtual nsresult GetFileForEntry(nsCacheEntry * entry, + nsIFile ** result); + + virtual nsresult OnDataSizeChange(nsCacheEntry * entry, int32_t deltaSize); + + virtual nsresult Visit(nsICacheVisitor * visitor); + + virtual nsresult EvictEntries(const char * clientID); + + bool EntryIsTooBig(int64_t entrySize); + + size_t SizeOfIncludingThis(mozilla::MallocSizeOf aMallocSizeOf); + + /** + * Preference accessors + */ + void SetCacheParentDirectory(nsIFile * parentDir); + void SetCapacity(uint32_t capacity); + void SetMaxEntrySize(int32_t maxSizeInKilobytes); + +/* private: */ + + void getCacheDirectory(nsIFile ** result); + uint32_t getCacheCapacity(); + uint32_t getCacheSize(); + uint32_t getEntryCount(); + + nsDiskCacheMap * CacheMap() { return &mCacheMap; } + +private: + friend class nsDiskCacheDeviceDeactivateEntryEvent; + friend class nsEvictDiskCacheEntriesEvent; + friend class nsDiskCacheMap; + /** + * Private methods + */ + + inline bool IsValidBinding(nsDiskCacheBinding *binding) + { + NS_ASSERTION(binding, " binding == nullptr"); + NS_ASSERTION(binding->mDeactivateEvent == nullptr, + " entry in process of deactivation"); + return (binding && !binding->mDeactivateEvent); + } + + bool Initialized() { return mInitialized; } + + nsresult Shutdown_Private(bool flush); + nsresult DeactivateEntry_Private(nsCacheEntry * entry, + nsDiskCacheBinding * binding); + + nsresult OpenDiskCache(); + nsresult ClearDiskCache(); + + nsresult EvictDiskCacheEntries(uint32_t targetCapacity); + + /** + * Member variables + */ + nsCOMPtr<nsIFile> mCacheDirectory; + nsDiskCacheBindery mBindery; + uint32_t mCacheCapacity; // Unit is KiB's + int32_t mMaxEntrySize; // Unit is bytes internally + // XXX need soft/hard limits, currentTotal + nsDiskCacheMap mCacheMap; + bool mInitialized; + bool mClearingDiskCache; +}; + +#endif // _nsDiskCacheDevice_h_ diff --git a/netwerk/cache/nsDiskCacheDeviceSQL.cpp b/netwerk/cache/nsDiskCacheDeviceSQL.cpp new file mode 100644 index 000000000..56ece5887 --- /dev/null +++ b/netwerk/cache/nsDiskCacheDeviceSQL.cpp @@ -0,0 +1,2906 @@ +/* -*- Mode: C++; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim:set ts=2 sw=2 sts=2 et cin: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include <inttypes.h> + +#include "mozilla/ArrayUtils.h" +#include "mozilla/Attributes.h" +#include "mozilla/Sprintf.h" +#include "mozilla/ThreadLocal.h" + +#include "nsCache.h" +#include "nsDiskCache.h" +#include "nsDiskCacheDeviceSQL.h" +#include "nsCacheService.h" +#include "nsApplicationCache.h" + +#include "nsNetCID.h" +#include "nsNetUtil.h" +#include "nsIURI.h" +#include "nsAutoPtr.h" +#include "nsEscape.h" +#include "nsIPrefBranch.h" +#include "nsIPrefService.h" +#include "nsString.h" +#include "nsPrintfCString.h" +#include "nsCRT.h" +#include "nsArrayUtils.h" +#include "nsIArray.h" +#include "nsIVariant.h" +#include "nsILoadContextInfo.h" +#include "nsThreadUtils.h" +#include "nsISerializable.h" +#include "nsIInputStream.h" +#include "nsIOutputStream.h" +#include "nsSerializationHelper.h" + +#include "mozIStorageService.h" +#include "mozIStorageStatement.h" +#include "mozIStorageFunction.h" +#include "mozStorageHelper.h" + +#include "nsICacheVisitor.h" +#include "nsISeekableStream.h" + +#include "mozilla/Telemetry.h" + +#include "sqlite3.h" +#include "mozilla/storage.h" +#include "nsVariant.h" +#include "mozilla/BasePrincipal.h" + +using namespace mozilla; +using namespace mozilla::storage; +using mozilla::NeckoOriginAttributes; + +static const char OFFLINE_CACHE_DEVICE_ID[] = { "offline" }; + +#define LOG(args) CACHE_LOG_DEBUG(args) + +static uint32_t gNextTemporaryClientID = 0; + +/***************************************************************************** + * helpers + */ + +static nsresult +EnsureDir(nsIFile *dir) +{ + bool exists; + nsresult rv = dir->Exists(&exists); + if (NS_SUCCEEDED(rv) && !exists) + rv = dir->Create(nsIFile::DIRECTORY_TYPE, 0700); + return rv; +} + +static bool +DecomposeCacheEntryKey(const nsCString *fullKey, + const char **cid, + const char **key, + nsCString &buf) +{ + buf = *fullKey; + + int32_t colon = buf.FindChar(':'); + if (colon == kNotFound) + { + NS_ERROR("Invalid key"); + return false; + } + buf.SetCharAt('\0', colon); + + *cid = buf.get(); + *key = buf.get() + colon + 1; + + return true; +} + +class AutoResetStatement +{ + public: + explicit AutoResetStatement(mozIStorageStatement *s) + : mStatement(s) {} + ~AutoResetStatement() { mStatement->Reset(); } + mozIStorageStatement *operator->() MOZ_NO_ADDREF_RELEASE_ON_RETURN { return mStatement; } + private: + mozIStorageStatement *mStatement; +}; + +class EvictionObserver +{ + public: + EvictionObserver(mozIStorageConnection *db, + nsOfflineCacheEvictionFunction *evictionFunction) + : mDB(db), mEvictionFunction(evictionFunction) + { + mEvictionFunction->Init(); + mDB->ExecuteSimpleSQL( + NS_LITERAL_CSTRING("CREATE TEMP TRIGGER cache_on_delete BEFORE DELETE" + " ON moz_cache FOR EACH ROW BEGIN SELECT" + " cache_eviction_observer(" + " OLD.ClientID, OLD.key, OLD.generation);" + " END;")); + } + + ~EvictionObserver() + { + mDB->ExecuteSimpleSQL( + NS_LITERAL_CSTRING("DROP TRIGGER cache_on_delete;")); + mEvictionFunction->Reset(); + } + + void Apply() { return mEvictionFunction->Apply(); } + + private: + mozIStorageConnection *mDB; + RefPtr<nsOfflineCacheEvictionFunction> mEvictionFunction; +}; + +#define DCACHE_HASH_MAX INT64_MAX +#define DCACHE_HASH_BITS 64 + +/** + * nsOfflineCache::Hash(const char * key) + * + * This algorithm of this method implies nsOfflineCacheRecords will be stored + * in a certain order on disk. If the algorithm changes, existing cache + * map files may become invalid, and therefore the kCurrentVersion needs + * to be revised. + */ +static uint64_t +DCacheHash(const char * key) +{ + // initval 0x7416f295 was chosen randomly + return (uint64_t(nsDiskCache::Hash(key, 0)) << 32) | nsDiskCache::Hash(key, 0x7416f295); +} + +/****************************************************************************** + * nsOfflineCacheEvictionFunction + */ + +NS_IMPL_ISUPPORTS(nsOfflineCacheEvictionFunction, mozIStorageFunction) + +// helper function for directly exposing the same data file binding +// path algorithm used in nsOfflineCacheBinding::Create +static nsresult +GetCacheDataFile(nsIFile *cacheDir, const char *key, + int generation, nsCOMPtr<nsIFile> &file) +{ + cacheDir->Clone(getter_AddRefs(file)); + if (!file) + return NS_ERROR_OUT_OF_MEMORY; + + uint64_t hash = DCacheHash(key); + + uint32_t dir1 = (uint32_t) (hash & 0x0F); + uint32_t dir2 = (uint32_t)((hash & 0xF0) >> 4); + + hash >>= 8; + + file->AppendNative(nsPrintfCString("%X", dir1)); + file->AppendNative(nsPrintfCString("%X", dir2)); + + char leaf[64]; + SprintfLiteral(leaf, "%014" PRIX64 "-%X", hash, generation); + return file->AppendNative(nsDependentCString(leaf)); +} + +namespace appcachedetail { + +typedef nsCOMArray<nsIFile> FileArray; +static MOZ_THREAD_LOCAL(FileArray*) tlsEvictionItems; + +} // appcachedetail + +NS_IMETHODIMP +nsOfflineCacheEvictionFunction::OnFunctionCall(mozIStorageValueArray *values, nsIVariant **_retval) +{ + LOG(("nsOfflineCacheEvictionFunction::OnFunctionCall\n")); + + *_retval = nullptr; + + uint32_t numEntries; + nsresult rv = values->GetNumEntries(&numEntries); + NS_ENSURE_SUCCESS(rv, rv); + NS_ASSERTION(numEntries == 3, "unexpected number of arguments"); + + uint32_t valueLen; + const char *clientID = values->AsSharedUTF8String(0, &valueLen); + const char *key = values->AsSharedUTF8String(1, &valueLen); + nsAutoCString fullKey(clientID); + fullKey.Append(':'); + fullKey.Append(key); + int generation = values->AsInt32(2); + + // If the key is currently locked, refuse to delete this row. + if (mDevice->IsLocked(fullKey)) { + NS_ADDREF(*_retval = new IntegerVariant(SQLITE_IGNORE)); + return NS_OK; + } + + nsCOMPtr<nsIFile> file; + rv = GetCacheDataFile(mDevice->CacheDirectory(), key, + generation, file); + if (NS_FAILED(rv)) + { + LOG(("GetCacheDataFile [key=%s generation=%d] failed [rv=%x]!\n", + key, generation, rv)); + return rv; + } + + appcachedetail::FileArray* items = appcachedetail::tlsEvictionItems.get(); + MOZ_ASSERT(items); + if (items) { + items->AppendObject(file); + } + + return NS_OK; +} + +nsOfflineCacheEvictionFunction::nsOfflineCacheEvictionFunction(nsOfflineCacheDevice * device) + : mDevice(device) +{ + mTLSInited = appcachedetail::tlsEvictionItems.init(); +} + +void nsOfflineCacheEvictionFunction::Init() +{ + if (mTLSInited) { + appcachedetail::tlsEvictionItems.set(new appcachedetail::FileArray()); + } +} + +void nsOfflineCacheEvictionFunction::Reset() +{ + if (!mTLSInited) { + return; + } + + appcachedetail::FileArray* items = appcachedetail::tlsEvictionItems.get(); + if (!items) { + return; + } + + appcachedetail::tlsEvictionItems.set(nullptr); + delete items; +} + +void +nsOfflineCacheEvictionFunction::Apply() +{ + LOG(("nsOfflineCacheEvictionFunction::Apply\n")); + + if (!mTLSInited) { + return; + } + + appcachedetail::FileArray* pitems = appcachedetail::tlsEvictionItems.get(); + if (!pitems) { + return; + } + + appcachedetail::FileArray items; + items.SwapElements(*pitems); + + for (int32_t i = 0; i < items.Count(); i++) { + if (MOZ_LOG_TEST(gCacheLog, LogLevel::Debug)) { + nsAutoCString path; + items[i]->GetNativePath(path); + LOG((" removing %s\n", path.get())); + } + + items[i]->Remove(false); + } +} + +class nsOfflineCacheDiscardCache : public Runnable +{ +public: + nsOfflineCacheDiscardCache(nsOfflineCacheDevice *device, + nsCString &group, + nsCString &clientID) + : mDevice(device) + , mGroup(group) + , mClientID(clientID) + { + } + + NS_IMETHOD Run() override + { + if (mDevice->IsActiveCache(mGroup, mClientID)) + { + mDevice->DeactivateGroup(mGroup); + } + + return mDevice->EvictEntries(mClientID.get()); + } + +private: + RefPtr<nsOfflineCacheDevice> mDevice; + nsCString mGroup; + nsCString mClientID; +}; + +/****************************************************************************** + * nsOfflineCacheDeviceInfo + */ + +class nsOfflineCacheDeviceInfo final : public nsICacheDeviceInfo +{ +public: + NS_DECL_ISUPPORTS + NS_DECL_NSICACHEDEVICEINFO + + explicit nsOfflineCacheDeviceInfo(nsOfflineCacheDevice* device) + : mDevice(device) + {} + +private: + ~nsOfflineCacheDeviceInfo() {} + + nsOfflineCacheDevice* mDevice; +}; + +NS_IMPL_ISUPPORTS(nsOfflineCacheDeviceInfo, nsICacheDeviceInfo) + +NS_IMETHODIMP +nsOfflineCacheDeviceInfo::GetDescription(char **aDescription) +{ + *aDescription = NS_strdup("Offline cache device"); + return *aDescription ? NS_OK : NS_ERROR_OUT_OF_MEMORY; +} + +NS_IMETHODIMP +nsOfflineCacheDeviceInfo::GetUsageReport(char ** usageReport) +{ + nsAutoCString buffer; + buffer.AssignLiteral(" <tr>\n" + " <th>Cache Directory:</th>\n" + " <td>"); + nsIFile *cacheDir = mDevice->CacheDirectory(); + if (!cacheDir) + return NS_OK; + + nsAutoString path; + nsresult rv = cacheDir->GetPath(path); + if (NS_SUCCEEDED(rv)) + AppendUTF16toUTF8(path, buffer); + else + buffer.AppendLiteral("directory unavailable"); + + buffer.AppendLiteral("</td>\n" + " </tr>\n"); + + *usageReport = ToNewCString(buffer); + if (!*usageReport) + return NS_ERROR_OUT_OF_MEMORY; + + return NS_OK; +} + +NS_IMETHODIMP +nsOfflineCacheDeviceInfo::GetEntryCount(uint32_t *aEntryCount) +{ + *aEntryCount = mDevice->EntryCount(); + return NS_OK; +} + +NS_IMETHODIMP +nsOfflineCacheDeviceInfo::GetTotalSize(uint32_t *aTotalSize) +{ + *aTotalSize = mDevice->CacheSize(); + return NS_OK; +} + +NS_IMETHODIMP +nsOfflineCacheDeviceInfo::GetMaximumSize(uint32_t *aMaximumSize) +{ + *aMaximumSize = mDevice->CacheCapacity(); + return NS_OK; +} + +/****************************************************************************** + * nsOfflineCacheBinding + */ + +class nsOfflineCacheBinding final : public nsISupports +{ + ~nsOfflineCacheBinding() {} + +public: + NS_DECL_THREADSAFE_ISUPPORTS + + static nsOfflineCacheBinding * + Create(nsIFile *cacheDir, const nsCString *key, int generation); + + enum { FLAG_NEW_ENTRY = 1 }; + + nsCOMPtr<nsIFile> mDataFile; + int mGeneration; + int mFlags; + + bool IsNewEntry() { return mFlags & FLAG_NEW_ENTRY; } + void MarkNewEntry() { mFlags |= FLAG_NEW_ENTRY; } + void ClearNewEntry() { mFlags &= ~FLAG_NEW_ENTRY; } +}; + +NS_IMPL_ISUPPORTS0(nsOfflineCacheBinding) + +nsOfflineCacheBinding * +nsOfflineCacheBinding::Create(nsIFile *cacheDir, + const nsCString *fullKey, + int generation) +{ + nsCOMPtr<nsIFile> file; + cacheDir->Clone(getter_AddRefs(file)); + if (!file) + return nullptr; + + nsAutoCString keyBuf; + const char *cid, *key; + if (!DecomposeCacheEntryKey(fullKey, &cid, &key, keyBuf)) + return nullptr; + + uint64_t hash = DCacheHash(key); + + uint32_t dir1 = (uint32_t) (hash & 0x0F); + uint32_t dir2 = (uint32_t)((hash & 0xF0) >> 4); + + hash >>= 8; + + // XXX we might want to create these directories up-front + + file->AppendNative(nsPrintfCString("%X", dir1)); + Unused << file->Create(nsIFile::DIRECTORY_TYPE, 00700); + + file->AppendNative(nsPrintfCString("%X", dir2)); + Unused << file->Create(nsIFile::DIRECTORY_TYPE, 00700); + + nsresult rv; + char leaf[64]; + + if (generation == -1) + { + file->AppendNative(NS_LITERAL_CSTRING("placeholder")); + + for (generation = 0; ; ++generation) + { + SprintfLiteral(leaf, "%014" PRIX64 "-%X", hash, generation); + + rv = file->SetNativeLeafName(nsDependentCString(leaf)); + if (NS_FAILED(rv)) + return nullptr; + rv = file->Create(nsIFile::NORMAL_FILE_TYPE, 00600); + if (NS_FAILED(rv) && rv != NS_ERROR_FILE_ALREADY_EXISTS) + return nullptr; + if (NS_SUCCEEDED(rv)) + break; + } + } + else + { + SprintfLiteral(leaf, "%014" PRIX64 "-%X", hash, generation); + rv = file->AppendNative(nsDependentCString(leaf)); + if (NS_FAILED(rv)) + return nullptr; + } + + nsOfflineCacheBinding *binding = new nsOfflineCacheBinding; + if (!binding) + return nullptr; + + binding->mDataFile.swap(file); + binding->mGeneration = generation; + binding->mFlags = 0; + return binding; +} + +/****************************************************************************** + * nsOfflineCacheRecord + */ + +struct nsOfflineCacheRecord +{ + const char *clientID; + const char *key; + const uint8_t *metaData; + uint32_t metaDataLen; + int32_t generation; + int32_t dataSize; + int32_t fetchCount; + int64_t lastFetched; + int64_t lastModified; + int64_t expirationTime; +}; + +static nsCacheEntry * +CreateCacheEntry(nsOfflineCacheDevice *device, + const nsCString *fullKey, + const nsOfflineCacheRecord &rec) +{ + nsCacheEntry *entry; + + if (device->IsLocked(*fullKey)) { + return nullptr; + } + + nsresult rv = nsCacheEntry::Create(fullKey->get(), // XXX enable sharing + nsICache::STREAM_BASED, + nsICache::STORE_OFFLINE, + device, &entry); + if (NS_FAILED(rv)) + return nullptr; + + entry->SetFetchCount((uint32_t) rec.fetchCount); + entry->SetLastFetched(SecondsFromPRTime(rec.lastFetched)); + entry->SetLastModified(SecondsFromPRTime(rec.lastModified)); + entry->SetExpirationTime(SecondsFromPRTime(rec.expirationTime)); + entry->SetDataSize((uint32_t) rec.dataSize); + + entry->UnflattenMetaData((const char *) rec.metaData, rec.metaDataLen); + + // Restore security info, if present + const char* info = entry->GetMetaDataElement("security-info"); + if (info) { + nsCOMPtr<nsISupports> infoObj; + rv = NS_DeserializeObject(nsDependentCString(info), + getter_AddRefs(infoObj)); + if (NS_FAILED(rv)) { + delete entry; + return nullptr; + } + entry->SetSecurityInfo(infoObj); + } + + // create a binding object for this entry + nsOfflineCacheBinding *binding = + nsOfflineCacheBinding::Create(device->CacheDirectory(), + fullKey, + rec.generation); + if (!binding) + { + delete entry; + return nullptr; + } + entry->SetData(binding); + + return entry; +} + + +/****************************************************************************** + * nsOfflineCacheEntryInfo + */ + +class nsOfflineCacheEntryInfo final : public nsICacheEntryInfo +{ + ~nsOfflineCacheEntryInfo() {} + +public: + NS_DECL_ISUPPORTS + NS_DECL_NSICACHEENTRYINFO + + nsOfflineCacheRecord *mRec; +}; + +NS_IMPL_ISUPPORTS(nsOfflineCacheEntryInfo, nsICacheEntryInfo) + +NS_IMETHODIMP +nsOfflineCacheEntryInfo::GetClientID(char **result) +{ + *result = NS_strdup(mRec->clientID); + return *result ? NS_OK : NS_ERROR_OUT_OF_MEMORY; +} + +NS_IMETHODIMP +nsOfflineCacheEntryInfo::GetDeviceID(char ** deviceID) +{ + *deviceID = NS_strdup(OFFLINE_CACHE_DEVICE_ID); + return *deviceID ? NS_OK : NS_ERROR_OUT_OF_MEMORY; +} + +NS_IMETHODIMP +nsOfflineCacheEntryInfo::GetKey(nsACString &clientKey) +{ + clientKey.Assign(mRec->key); + return NS_OK; +} + +NS_IMETHODIMP +nsOfflineCacheEntryInfo::GetFetchCount(int32_t *aFetchCount) +{ + *aFetchCount = mRec->fetchCount; + return NS_OK; +} + +NS_IMETHODIMP +nsOfflineCacheEntryInfo::GetLastFetched(uint32_t *aLastFetched) +{ + *aLastFetched = SecondsFromPRTime(mRec->lastFetched); + return NS_OK; +} + +NS_IMETHODIMP +nsOfflineCacheEntryInfo::GetLastModified(uint32_t *aLastModified) +{ + *aLastModified = SecondsFromPRTime(mRec->lastModified); + return NS_OK; +} + +NS_IMETHODIMP +nsOfflineCacheEntryInfo::GetExpirationTime(uint32_t *aExpirationTime) +{ + *aExpirationTime = SecondsFromPRTime(mRec->expirationTime); + return NS_OK; +} + +NS_IMETHODIMP +nsOfflineCacheEntryInfo::IsStreamBased(bool *aStreamBased) +{ + *aStreamBased = true; + return NS_OK; +} + +NS_IMETHODIMP +nsOfflineCacheEntryInfo::GetDataSize(uint32_t *aDataSize) +{ + *aDataSize = mRec->dataSize; + return NS_OK; +} + + +/****************************************************************************** + * nsApplicationCacheNamespace + */ + +NS_IMPL_ISUPPORTS(nsApplicationCacheNamespace, nsIApplicationCacheNamespace) + +NS_IMETHODIMP +nsApplicationCacheNamespace::Init(uint32_t itemType, + const nsACString &namespaceSpec, + const nsACString &data) +{ + mItemType = itemType; + mNamespaceSpec = namespaceSpec; + mData = data; + return NS_OK; +} + +NS_IMETHODIMP +nsApplicationCacheNamespace::GetItemType(uint32_t *out) +{ + *out = mItemType; + return NS_OK; +} + +NS_IMETHODIMP +nsApplicationCacheNamespace::GetNamespaceSpec(nsACString &out) +{ + out = mNamespaceSpec; + return NS_OK; +} + +NS_IMETHODIMP +nsApplicationCacheNamespace::GetData(nsACString &out) +{ + out = mData; + return NS_OK; +} + +/****************************************************************************** + * nsApplicationCache + */ + +NS_IMPL_ISUPPORTS(nsApplicationCache, + nsIApplicationCache, + nsISupportsWeakReference) + +nsApplicationCache::nsApplicationCache() + : mDevice(nullptr) + , mValid(true) +{ +} + +nsApplicationCache::nsApplicationCache(nsOfflineCacheDevice *device, + const nsACString &group, + const nsACString &clientID) + : mDevice(device) + , mGroup(group) + , mClientID(clientID) + , mValid(true) +{ +} + +nsApplicationCache::~nsApplicationCache() +{ + if (!mDevice) + return; + + { + MutexAutoLock lock(mDevice->mLock); + mDevice->mCaches.Remove(mClientID); + } + + // If this isn't an active cache anymore, it can be destroyed. + if (mValid && !mDevice->IsActiveCache(mGroup, mClientID)) + Discard(); +} + +void +nsApplicationCache::MarkInvalid() +{ + mValid = false; +} + +NS_IMETHODIMP +nsApplicationCache::InitAsHandle(const nsACString &groupId, + const nsACString &clientId) +{ + NS_ENSURE_FALSE(mDevice, NS_ERROR_ALREADY_INITIALIZED); + NS_ENSURE_TRUE(mGroup.IsEmpty(), NS_ERROR_ALREADY_INITIALIZED); + + mGroup = groupId; + mClientID = clientId; + return NS_OK; +} + +NS_IMETHODIMP +nsApplicationCache::GetManifestURI(nsIURI **out) +{ + nsCOMPtr<nsIURI> uri; + nsresult rv = NS_NewURI(getter_AddRefs(uri), mGroup); + NS_ENSURE_SUCCESS(rv, rv); + + rv = uri->CloneIgnoringRef(out); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +NS_IMETHODIMP +nsApplicationCache::GetGroupID(nsACString &out) +{ + out = mGroup; + return NS_OK; +} + +NS_IMETHODIMP +nsApplicationCache::GetClientID(nsACString &out) +{ + out = mClientID; + return NS_OK; +} + +NS_IMETHODIMP +nsApplicationCache::GetProfileDirectory(nsIFile **out) +{ + if (mDevice->BaseDirectory()) + NS_ADDREF(*out = mDevice->BaseDirectory()); + else + *out = nullptr; + + return NS_OK; +} + +NS_IMETHODIMP +nsApplicationCache::GetActive(bool *out) +{ + NS_ENSURE_TRUE(mDevice, NS_ERROR_NOT_AVAILABLE); + + *out = mDevice->IsActiveCache(mGroup, mClientID); + return NS_OK; +} + +NS_IMETHODIMP +nsApplicationCache::Activate() +{ + NS_ENSURE_TRUE(mValid, NS_ERROR_NOT_AVAILABLE); + NS_ENSURE_TRUE(mDevice, NS_ERROR_NOT_AVAILABLE); + + mDevice->ActivateCache(mGroup, mClientID); + + if (mDevice->AutoShutdown(this)) + mDevice = nullptr; + + return NS_OK; +} + +NS_IMETHODIMP +nsApplicationCache::Discard() +{ + NS_ENSURE_TRUE(mValid, NS_ERROR_NOT_AVAILABLE); + NS_ENSURE_TRUE(mDevice, NS_ERROR_NOT_AVAILABLE); + + mValid = false; + + nsCOMPtr<nsIRunnable> ev = + new nsOfflineCacheDiscardCache(mDevice, mGroup, mClientID); + nsresult rv = nsCacheService::DispatchToCacheIOThread(ev); + return rv; +} + +NS_IMETHODIMP +nsApplicationCache::MarkEntry(const nsACString &key, + uint32_t typeBits) +{ + NS_ENSURE_TRUE(mValid, NS_ERROR_NOT_AVAILABLE); + NS_ENSURE_TRUE(mDevice, NS_ERROR_NOT_AVAILABLE); + + return mDevice->MarkEntry(mClientID, key, typeBits); +} + + +NS_IMETHODIMP +nsApplicationCache::UnmarkEntry(const nsACString &key, + uint32_t typeBits) +{ + NS_ENSURE_TRUE(mValid, NS_ERROR_NOT_AVAILABLE); + NS_ENSURE_TRUE(mDevice, NS_ERROR_NOT_AVAILABLE); + + return mDevice->UnmarkEntry(mClientID, key, typeBits); +} + +NS_IMETHODIMP +nsApplicationCache::GetTypes(const nsACString &key, + uint32_t *typeBits) +{ + NS_ENSURE_TRUE(mValid, NS_ERROR_NOT_AVAILABLE); + NS_ENSURE_TRUE(mDevice, NS_ERROR_NOT_AVAILABLE); + + return mDevice->GetTypes(mClientID, key, typeBits); +} + +NS_IMETHODIMP +nsApplicationCache::GatherEntries(uint32_t typeBits, + uint32_t * count, + char *** keys) +{ + NS_ENSURE_TRUE(mValid, NS_ERROR_NOT_AVAILABLE); + NS_ENSURE_TRUE(mDevice, NS_ERROR_NOT_AVAILABLE); + + return mDevice->GatherEntries(mClientID, typeBits, count, keys); +} + +NS_IMETHODIMP +nsApplicationCache::AddNamespaces(nsIArray *namespaces) +{ + NS_ENSURE_TRUE(mValid, NS_ERROR_NOT_AVAILABLE); + NS_ENSURE_TRUE(mDevice, NS_ERROR_NOT_AVAILABLE); + + if (!namespaces) + return NS_OK; + + mozStorageTransaction transaction(mDevice->mDB, false); + + uint32_t length; + nsresult rv = namespaces->GetLength(&length); + NS_ENSURE_SUCCESS(rv, rv); + + for (uint32_t i = 0; i < length; i++) { + nsCOMPtr<nsIApplicationCacheNamespace> ns = + do_QueryElementAt(namespaces, i); + if (ns) { + rv = mDevice->AddNamespace(mClientID, ns); + NS_ENSURE_SUCCESS(rv, rv); + } + } + + rv = transaction.Commit(); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +NS_IMETHODIMP +nsApplicationCache::GetMatchingNamespace(const nsACString &key, + nsIApplicationCacheNamespace **out) + +{ + NS_ENSURE_TRUE(mValid, NS_ERROR_NOT_AVAILABLE); + NS_ENSURE_TRUE(mDevice, NS_ERROR_NOT_AVAILABLE); + + return mDevice->GetMatchingNamespace(mClientID, key, out); +} + +NS_IMETHODIMP +nsApplicationCache::GetUsage(uint32_t *usage) +{ + NS_ENSURE_TRUE(mValid, NS_ERROR_NOT_AVAILABLE); + NS_ENSURE_TRUE(mDevice, NS_ERROR_NOT_AVAILABLE); + + return mDevice->GetUsage(mClientID, usage); +} + +/****************************************************************************** + * nsCloseDBEvent + *****************************************************************************/ + +class nsCloseDBEvent : public Runnable { +public: + explicit nsCloseDBEvent(mozIStorageConnection *aDB) + { + mDB = aDB; + } + + NS_IMETHOD Run() override + { + mDB->Close(); + return NS_OK; + } + +protected: + virtual ~nsCloseDBEvent() {} + +private: + nsCOMPtr<mozIStorageConnection> mDB; +}; + + + +/****************************************************************************** + * nsOfflineCacheDevice + */ + +NS_IMPL_ISUPPORTS0(nsOfflineCacheDevice) + +nsOfflineCacheDevice::nsOfflineCacheDevice() + : mDB(nullptr) + , mCacheCapacity(0) + , mDeltaCounter(0) + , mAutoShutdown(false) + , mLock("nsOfflineCacheDevice.lock") + , mActiveCaches(4) + , mLockedEntries(32) +{ +} + +nsOfflineCacheDevice::~nsOfflineCacheDevice() +{} + +/* static */ +bool +nsOfflineCacheDevice::GetStrictFileOriginPolicy() +{ + nsCOMPtr<nsIPrefBranch> prefs = do_GetService(NS_PREFSERVICE_CONTRACTID); + + bool retval; + if (prefs && NS_SUCCEEDED(prefs->GetBoolPref("security.fileuri.strict_origin_policy", &retval))) + return retval; + + // As default value use true (be more strict) + return true; +} + +uint32_t +nsOfflineCacheDevice::CacheSize() +{ + NS_ENSURE_TRUE(Initialized(), 0); + + AutoResetStatement statement(mStatement_CacheSize); + + bool hasRows; + nsresult rv = statement->ExecuteStep(&hasRows); + NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && hasRows, 0); + + return (uint32_t) statement->AsInt32(0); +} + +uint32_t +nsOfflineCacheDevice::EntryCount() +{ + NS_ENSURE_TRUE(Initialized(), 0); + + AutoResetStatement statement(mStatement_EntryCount); + + bool hasRows; + nsresult rv = statement->ExecuteStep(&hasRows); + NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && hasRows, 0); + + return (uint32_t) statement->AsInt32(0); +} + +nsresult +nsOfflineCacheDevice::UpdateEntry(nsCacheEntry *entry) +{ + NS_ENSURE_TRUE(Initialized(), NS_ERROR_NOT_INITIALIZED); + + // Decompose the key into "ClientID" and "Key" + nsAutoCString keyBuf; + const char *cid, *key; + + if (!DecomposeCacheEntryKey(entry->Key(), &cid, &key, keyBuf)) + return NS_ERROR_UNEXPECTED; + + // Store security info, if it is serializable + nsCOMPtr<nsISupports> infoObj = entry->SecurityInfo(); + nsCOMPtr<nsISerializable> serializable = do_QueryInterface(infoObj); + if (infoObj && !serializable) + return NS_ERROR_UNEXPECTED; + + if (serializable) { + nsCString info; + nsresult rv = NS_SerializeToString(serializable, info); + NS_ENSURE_SUCCESS(rv, rv); + + rv = entry->SetMetaDataElement("security-info", info.get()); + NS_ENSURE_SUCCESS(rv, rv); + } + + nsCString metaDataBuf; + uint32_t mdSize = entry->MetaDataSize(); + if (!metaDataBuf.SetLength(mdSize, fallible)) + return NS_ERROR_OUT_OF_MEMORY; + char *md = metaDataBuf.BeginWriting(); + entry->FlattenMetaData(md, mdSize); + + nsOfflineCacheRecord rec; + rec.metaData = (const uint8_t *) md; + rec.metaDataLen = mdSize; + rec.dataSize = entry->DataSize(); + rec.fetchCount = entry->FetchCount(); + rec.lastFetched = PRTimeFromSeconds(entry->LastFetched()); + rec.lastModified = PRTimeFromSeconds(entry->LastModified()); + rec.expirationTime = PRTimeFromSeconds(entry->ExpirationTime()); + + AutoResetStatement statement(mStatement_UpdateEntry); + + nsresult rv; + rv = statement->BindBlobByIndex(0, rec.metaData, rec.metaDataLen); + nsresult tmp = statement->BindInt32ByIndex(1, rec.dataSize); + if (NS_FAILED(tmp)) { + rv = tmp; + } + tmp = statement->BindInt32ByIndex(2, rec.fetchCount); + if (NS_FAILED(tmp)) { + rv = tmp; + } + tmp = statement->BindInt64ByIndex(3, rec.lastFetched); + if (NS_FAILED(tmp)) { + rv = tmp; + } + tmp = statement->BindInt64ByIndex(4, rec.lastModified); + if (NS_FAILED(tmp)) { + rv = tmp; + } + tmp = statement->BindInt64ByIndex(5, rec.expirationTime); + if (NS_FAILED(tmp)) { + rv = tmp; + } + tmp = statement->BindUTF8StringByIndex(6, nsDependentCString(cid)); + if (NS_FAILED(tmp)) { + rv = tmp; + } + tmp = statement->BindUTF8StringByIndex(7, nsDependentCString(key)); + if (NS_FAILED(tmp)) { + rv = tmp; + } + NS_ENSURE_SUCCESS(rv, rv); + + bool hasRows; + rv = statement->ExecuteStep(&hasRows); + NS_ENSURE_SUCCESS(rv, rv); + + NS_ASSERTION(!hasRows, "UPDATE should not result in output"); + return rv; +} + +nsresult +nsOfflineCacheDevice::UpdateEntrySize(nsCacheEntry *entry, uint32_t newSize) +{ + NS_ENSURE_TRUE(Initialized(), NS_ERROR_NOT_INITIALIZED); + + // Decompose the key into "ClientID" and "Key" + nsAutoCString keyBuf; + const char *cid, *key; + if (!DecomposeCacheEntryKey(entry->Key(), &cid, &key, keyBuf)) + return NS_ERROR_UNEXPECTED; + + AutoResetStatement statement(mStatement_UpdateEntrySize); + + nsresult rv = statement->BindInt32ByIndex(0, newSize); + nsresult tmp = statement->BindUTF8StringByIndex(1, nsDependentCString(cid)); + if (NS_FAILED(tmp)) { + rv = tmp; + } + tmp = statement->BindUTF8StringByIndex(2, nsDependentCString(key)); + if (NS_FAILED(tmp)) { + rv = tmp; + } + NS_ENSURE_SUCCESS(rv, rv); + + bool hasRows; + rv = statement->ExecuteStep(&hasRows); + NS_ENSURE_SUCCESS(rv, rv); + + NS_ASSERTION(!hasRows, "UPDATE should not result in output"); + return rv; +} + +nsresult +nsOfflineCacheDevice::DeleteEntry(nsCacheEntry *entry, bool deleteData) +{ + NS_ENSURE_TRUE(Initialized(), NS_ERROR_NOT_INITIALIZED); + + if (deleteData) + { + nsresult rv = DeleteData(entry); + if (NS_FAILED(rv)) + return rv; + } + + // Decompose the key into "ClientID" and "Key" + nsAutoCString keyBuf; + const char *cid, *key; + if (!DecomposeCacheEntryKey(entry->Key(), &cid, &key, keyBuf)) + return NS_ERROR_UNEXPECTED; + + AutoResetStatement statement(mStatement_DeleteEntry); + + nsresult rv = statement->BindUTF8StringByIndex(0, nsDependentCString(cid)); + nsresult rv2 = statement->BindUTF8StringByIndex(1, nsDependentCString(key)); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_SUCCESS(rv2, rv2); + + bool hasRows; + rv = statement->ExecuteStep(&hasRows); + NS_ENSURE_SUCCESS(rv, rv); + + NS_ASSERTION(!hasRows, "DELETE should not result in output"); + return rv; +} + +nsresult +nsOfflineCacheDevice::DeleteData(nsCacheEntry *entry) +{ + nsOfflineCacheBinding *binding = (nsOfflineCacheBinding *) entry->Data(); + NS_ENSURE_STATE(binding); + + return binding->mDataFile->Remove(false); +} + +/** + * nsCacheDevice implementation + */ + +// This struct is local to nsOfflineCacheDevice::Init, but ISO C++98 doesn't +// allow a template (mozilla::ArrayLength) to be instantiated based on a local +// type. Boo-urns! +struct StatementSql { + nsCOMPtr<mozIStorageStatement> &statement; + const char *sql; + StatementSql (nsCOMPtr<mozIStorageStatement> &aStatement, const char *aSql): + statement (aStatement), sql (aSql) {} +}; + +nsresult +nsOfflineCacheDevice::Init() +{ + MOZ_ASSERT(false, "Need to be initialized with sqlite"); + return NS_ERROR_NOT_IMPLEMENTED; +} + +nsresult +nsOfflineCacheDevice::InitWithSqlite(mozIStorageService * ss) +{ + NS_ENSURE_TRUE(!mDB, NS_ERROR_ALREADY_INITIALIZED); + + // SetCacheParentDirectory must have been called + NS_ENSURE_TRUE(mCacheDirectory, NS_ERROR_UNEXPECTED); + + // make sure the cache directory exists + nsresult rv = EnsureDir(mCacheDirectory); + NS_ENSURE_SUCCESS(rv, rv); + + // build path to index file + nsCOMPtr<nsIFile> indexFile; + rv = mCacheDirectory->Clone(getter_AddRefs(indexFile)); + NS_ENSURE_SUCCESS(rv, rv); + rv = indexFile->AppendNative(NS_LITERAL_CSTRING("index.sqlite")); + NS_ENSURE_SUCCESS(rv, rv); + + MOZ_ASSERT(ss, "nsOfflineCacheDevice::InitWithSqlite called before nsCacheService::Init() ?"); + NS_ENSURE_TRUE(ss, NS_ERROR_UNEXPECTED); + + rv = ss->OpenDatabase(indexFile, getter_AddRefs(mDB)); + NS_ENSURE_SUCCESS(rv, rv); + + mInitThread = do_GetCurrentThread(); + + mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("PRAGMA synchronous = OFF;")); + + // XXX ... other initialization steps + + // XXX in the future we may wish to verify the schema for moz_cache + // perhaps using "PRAGMA table_info" ? + + // build the table + // + // "Generation" is the data file generation number. + // + rv = mDB->ExecuteSimpleSQL( + NS_LITERAL_CSTRING("CREATE TABLE IF NOT EXISTS moz_cache (\n" + " ClientID TEXT,\n" + " Key TEXT,\n" + " MetaData BLOB,\n" + " Generation INTEGER,\n" + " DataSize INTEGER,\n" + " FetchCount INTEGER,\n" + " LastFetched INTEGER,\n" + " LastModified INTEGER,\n" + " ExpirationTime INTEGER,\n" + " ItemType INTEGER DEFAULT 0\n" + ");\n")); + NS_ENSURE_SUCCESS(rv, rv); + + // Databases from 1.9.0 don't have the ItemType column. Add the column + // here, but don't worry about failures (the column probably already exists) + mDB->ExecuteSimpleSQL( + NS_LITERAL_CSTRING("ALTER TABLE moz_cache ADD ItemType INTEGER DEFAULT 0")); + + // Create the table for storing cache groups. All actions on + // moz_cache_groups use the GroupID, so use it as the primary key. + rv = mDB->ExecuteSimpleSQL( + NS_LITERAL_CSTRING("CREATE TABLE IF NOT EXISTS moz_cache_groups (\n" + " GroupID TEXT PRIMARY KEY,\n" + " ActiveClientID TEXT\n" + ");\n")); + NS_ENSURE_SUCCESS(rv, rv); + + mDB->ExecuteSimpleSQL( + NS_LITERAL_CSTRING("ALTER TABLE moz_cache_groups " + "ADD ActivateTimeStamp INTEGER DEFAULT 0")); + + // ClientID: clientID joining moz_cache and moz_cache_namespaces + // tables. + // Data: Data associated with this namespace (e.g. a fallback URI + // for fallback entries). + // ItemType: the type of namespace. + rv = mDB->ExecuteSimpleSQL( + NS_LITERAL_CSTRING("CREATE TABLE IF NOT EXISTS" + " moz_cache_namespaces (\n" + " ClientID TEXT,\n" + " NameSpace TEXT,\n" + " Data TEXT,\n" + " ItemType INTEGER\n" + ");\n")); + NS_ENSURE_SUCCESS(rv, rv); + + // Databases from 1.9.0 have a moz_cache_index that should be dropped + rv = mDB->ExecuteSimpleSQL( + NS_LITERAL_CSTRING("DROP INDEX IF EXISTS moz_cache_index")); + NS_ENSURE_SUCCESS(rv, rv); + + // Key/ClientID pairs should be unique in the database. All queries + // against moz_cache use the Key (which is also the most unique), so + // use it as the primary key for this index. + rv = mDB->ExecuteSimpleSQL( + NS_LITERAL_CSTRING("CREATE UNIQUE INDEX IF NOT EXISTS " + " moz_cache_key_clientid_index" + " ON moz_cache (Key, ClientID);")); + NS_ENSURE_SUCCESS(rv, rv); + + // Used for ClientID lookups and to keep ClientID/NameSpace pairs unique. + rv = mDB->ExecuteSimpleSQL( + NS_LITERAL_CSTRING("CREATE UNIQUE INDEX IF NOT EXISTS" + " moz_cache_namespaces_clientid_index" + " ON moz_cache_namespaces (ClientID, NameSpace);")); + NS_ENSURE_SUCCESS(rv, rv); + + // Used for namespace lookups. + rv = mDB->ExecuteSimpleSQL( + NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS" + " moz_cache_namespaces_namespace_index" + " ON moz_cache_namespaces (NameSpace);")); + NS_ENSURE_SUCCESS(rv, rv); + + + mEvictionFunction = new nsOfflineCacheEvictionFunction(this); + if (!mEvictionFunction) return NS_ERROR_OUT_OF_MEMORY; + + rv = mDB->CreateFunction(NS_LITERAL_CSTRING("cache_eviction_observer"), 3, mEvictionFunction); + NS_ENSURE_SUCCESS(rv, rv); + + // create all (most) of our statements up front + StatementSql prepared[] = { + StatementSql ( mStatement_CacheSize, "SELECT Sum(DataSize) from moz_cache;" ), + StatementSql ( mStatement_ApplicationCacheSize, "SELECT Sum(DataSize) from moz_cache WHERE ClientID = ?;" ), + StatementSql ( mStatement_EntryCount, "SELECT count(*) from moz_cache;" ), + StatementSql ( mStatement_UpdateEntry, "UPDATE moz_cache SET MetaData = ?, DataSize = ?, FetchCount = ?, LastFetched = ?, LastModified = ?, ExpirationTime = ? WHERE ClientID = ? AND Key = ?;" ), + StatementSql ( mStatement_UpdateEntrySize, "UPDATE moz_cache SET DataSize = ? WHERE ClientID = ? AND Key = ?;" ), + StatementSql ( mStatement_DeleteEntry, "DELETE FROM moz_cache WHERE ClientID = ? AND Key = ?;" ), + StatementSql ( mStatement_FindEntry, "SELECT MetaData, Generation, DataSize, FetchCount, LastFetched, LastModified, ExpirationTime, ItemType FROM moz_cache WHERE ClientID = ? AND Key = ?;" ), + StatementSql ( mStatement_BindEntry, "INSERT INTO moz_cache (ClientID, Key, MetaData, Generation, DataSize, FetchCount, LastFetched, LastModified, ExpirationTime) VALUES(?,?,?,?,?,?,?,?,?);" ), + + StatementSql ( mStatement_MarkEntry, "UPDATE moz_cache SET ItemType = (ItemType | ?) WHERE ClientID = ? AND Key = ?;" ), + StatementSql ( mStatement_UnmarkEntry, "UPDATE moz_cache SET ItemType = (ItemType & ~?) WHERE ClientID = ? AND Key = ?;" ), + StatementSql ( mStatement_GetTypes, "SELECT ItemType FROM moz_cache WHERE ClientID = ? AND Key = ?;"), + StatementSql ( mStatement_CleanupUnmarked, "DELETE FROM moz_cache WHERE ClientID = ? AND Key = ? AND ItemType = 0;" ), + StatementSql ( mStatement_GatherEntries, "SELECT Key FROM moz_cache WHERE ClientID = ? AND (ItemType & ?) > 0;" ), + + StatementSql ( mStatement_ActivateClient, "INSERT OR REPLACE INTO moz_cache_groups (GroupID, ActiveClientID, ActivateTimeStamp) VALUES (?, ?, ?);" ), + StatementSql ( mStatement_DeactivateGroup, "DELETE FROM moz_cache_groups WHERE GroupID = ?;" ), + StatementSql ( mStatement_FindClient, "SELECT ClientID, ItemType FROM moz_cache WHERE Key = ? ORDER BY LastFetched DESC, LastModified DESC;" ), + + // Search for namespaces that match the URI. Use the <= operator + // to ensure that we use the index on moz_cache_namespaces. + StatementSql ( mStatement_FindClientByNamespace, "SELECT ns.ClientID, ns.ItemType FROM" + " moz_cache_namespaces AS ns JOIN moz_cache_groups AS groups" + " ON ns.ClientID = groups.ActiveClientID" + " WHERE ns.NameSpace <= ?1 AND ?1 GLOB ns.NameSpace || '*'" + " ORDER BY ns.NameSpace DESC, groups.ActivateTimeStamp DESC;"), + StatementSql ( mStatement_FindNamespaceEntry, "SELECT NameSpace, Data, ItemType FROM moz_cache_namespaces" + " WHERE ClientID = ?1" + " AND NameSpace <= ?2 AND ?2 GLOB NameSpace || '*'" + " ORDER BY NameSpace DESC;"), + StatementSql ( mStatement_InsertNamespaceEntry, "INSERT INTO moz_cache_namespaces (ClientID, NameSpace, Data, ItemType) VALUES(?, ?, ?, ?);"), + StatementSql ( mStatement_EnumerateApps, "SELECT GroupID, ActiveClientID FROM moz_cache_groups WHERE GroupID LIKE ?1;"), + StatementSql ( mStatement_EnumerateGroups, "SELECT GroupID, ActiveClientID FROM moz_cache_groups;"), + StatementSql ( mStatement_EnumerateGroupsTimeOrder, "SELECT GroupID, ActiveClientID FROM moz_cache_groups ORDER BY ActivateTimeStamp;") + }; + for (uint32_t i = 0; NS_SUCCEEDED(rv) && i < ArrayLength(prepared); ++i) + { + LOG(("Creating statement: %s\n", prepared[i].sql)); + + rv = mDB->CreateStatement(nsDependentCString(prepared[i].sql), + getter_AddRefs(prepared[i].statement)); + NS_ENSURE_SUCCESS(rv, rv); + } + + rv = InitActiveCaches(); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +namespace { + +nsresult +GetGroupForCache(const nsCSubstring &clientID, nsCString &group) +{ + group.Assign(clientID); + group.Truncate(group.FindChar('|')); + NS_UnescapeURL(group); + + return NS_OK; +} + +} // namespace + +// static +nsresult +nsOfflineCacheDevice::BuildApplicationCacheGroupID(nsIURI *aManifestURL, + nsACString const &aOriginSuffix, + nsACString &_result) +{ + nsCOMPtr<nsIURI> newURI; + nsresult rv = aManifestURL->CloneIgnoringRef(getter_AddRefs(newURI)); + NS_ENSURE_SUCCESS(rv, rv); + + nsAutoCString manifestSpec; + rv = newURI->GetAsciiSpec(manifestSpec); + NS_ENSURE_SUCCESS(rv, rv); + + _result.Assign(manifestSpec); + _result.Append('#'); + _result.Append(aOriginSuffix); + + return NS_OK; +} + +nsresult +nsOfflineCacheDevice::InitActiveCaches() +{ + NS_ENSURE_TRUE(Initialized(), NS_ERROR_NOT_INITIALIZED); + + MutexAutoLock lock(mLock); + + AutoResetStatement statement(mStatement_EnumerateGroups); + + bool hasRows; + nsresult rv = statement->ExecuteStep(&hasRows); + NS_ENSURE_SUCCESS(rv, rv); + + while (hasRows) + { + nsAutoCString group; + statement->GetUTF8String(0, group); + nsCString clientID; + statement->GetUTF8String(1, clientID); + + mActiveCaches.PutEntry(clientID); + mActiveCachesByGroup.Put(group, new nsCString(clientID)); + + rv = statement->ExecuteStep(&hasRows); + NS_ENSURE_SUCCESS(rv, rv); + } + + return NS_OK; +} + +nsresult +nsOfflineCacheDevice::Shutdown() +{ + NS_ENSURE_TRUE(mDB, NS_ERROR_NOT_INITIALIZED); + + { + MutexAutoLock lock(mLock); + for (auto iter = mCaches.Iter(); !iter.Done(); iter.Next()) { + nsCOMPtr<nsIApplicationCache> obj = do_QueryReferent(iter.UserData()); + if (obj) { + auto appCache = static_cast<nsApplicationCache*>(obj.get()); + appCache->MarkInvalid(); + } + } + } + + { + EvictionObserver evictionObserver(mDB, mEvictionFunction); + + // Delete all rows whose clientID is not an active clientID. + nsresult rv = mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "DELETE FROM moz_cache WHERE rowid IN" + " (SELECT moz_cache.rowid FROM" + " moz_cache LEFT OUTER JOIN moz_cache_groups ON" + " (moz_cache.ClientID = moz_cache_groups.ActiveClientID)" + " WHERE moz_cache_groups.GroupID ISNULL)")); + + if (NS_FAILED(rv)) + NS_WARNING("Failed to clean up unused application caches."); + else + evictionObserver.Apply(); + + // Delete all namespaces whose clientID is not an active clientID. + rv = mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "DELETE FROM moz_cache_namespaces WHERE rowid IN" + " (SELECT moz_cache_namespaces.rowid FROM" + " moz_cache_namespaces LEFT OUTER JOIN moz_cache_groups ON" + " (moz_cache_namespaces.ClientID = moz_cache_groups.ActiveClientID)" + " WHERE moz_cache_groups.GroupID ISNULL)")); + + if (NS_FAILED(rv)) + NS_WARNING("Failed to clean up namespaces."); + + mEvictionFunction = nullptr; + + mStatement_CacheSize = nullptr; + mStatement_ApplicationCacheSize = nullptr; + mStatement_EntryCount = nullptr; + mStatement_UpdateEntry = nullptr; + mStatement_UpdateEntrySize = nullptr; + mStatement_DeleteEntry = nullptr; + mStatement_FindEntry = nullptr; + mStatement_BindEntry = nullptr; + mStatement_ClearDomain = nullptr; + mStatement_MarkEntry = nullptr; + mStatement_UnmarkEntry = nullptr; + mStatement_GetTypes = nullptr; + mStatement_FindNamespaceEntry = nullptr; + mStatement_InsertNamespaceEntry = nullptr; + mStatement_CleanupUnmarked = nullptr; + mStatement_GatherEntries = nullptr; + mStatement_ActivateClient = nullptr; + mStatement_DeactivateGroup = nullptr; + mStatement_FindClient = nullptr; + mStatement_FindClientByNamespace = nullptr; + mStatement_EnumerateApps = nullptr; + mStatement_EnumerateGroups = nullptr; + mStatement_EnumerateGroupsTimeOrder = nullptr; + } + + // Close Database on the correct thread + bool isOnCurrentThread = true; + if (mInitThread) + mInitThread->IsOnCurrentThread(&isOnCurrentThread); + + if (!isOnCurrentThread) { + nsCOMPtr<nsIRunnable> ev = new nsCloseDBEvent(mDB); + + if (ev) { + mInitThread->Dispatch(ev, NS_DISPATCH_NORMAL); + } + } + else { + mDB->Close(); + } + + mDB = nullptr; + mInitThread = nullptr; + + return NS_OK; +} + +const char * +nsOfflineCacheDevice::GetDeviceID() +{ + return OFFLINE_CACHE_DEVICE_ID; +} + +nsCacheEntry * +nsOfflineCacheDevice::FindEntry(nsCString *fullKey, bool *collision) +{ + NS_ENSURE_TRUE(Initialized(), nullptr); + + mozilla::Telemetry::AutoTimer<mozilla::Telemetry::CACHE_OFFLINE_SEARCH_2> timer; + LOG(("nsOfflineCacheDevice::FindEntry [key=%s]\n", fullKey->get())); + + // SELECT * FROM moz_cache WHERE key = ? + + // Decompose the key into "ClientID" and "Key" + nsAutoCString keyBuf; + const char *cid, *key; + if (!DecomposeCacheEntryKey(fullKey, &cid, &key, keyBuf)) + return nullptr; + + AutoResetStatement statement(mStatement_FindEntry); + + nsresult rv = statement->BindUTF8StringByIndex(0, nsDependentCString(cid)); + nsresult rv2 = statement->BindUTF8StringByIndex(1, nsDependentCString(key)); + NS_ENSURE_SUCCESS(rv, nullptr); + NS_ENSURE_SUCCESS(rv2, nullptr); + + bool hasRows; + rv = statement->ExecuteStep(&hasRows); + if (NS_FAILED(rv) || !hasRows) + return nullptr; // entry not found + + nsOfflineCacheRecord rec; + statement->GetSharedBlob(0, &rec.metaDataLen, + (const uint8_t **) &rec.metaData); + rec.generation = statement->AsInt32(1); + rec.dataSize = statement->AsInt32(2); + rec.fetchCount = statement->AsInt32(3); + rec.lastFetched = statement->AsInt64(4); + rec.lastModified = statement->AsInt64(5); + rec.expirationTime = statement->AsInt64(6); + + LOG(("entry: [%u %d %d %d %lld %lld %lld]\n", + rec.metaDataLen, + rec.generation, + rec.dataSize, + rec.fetchCount, + rec.lastFetched, + rec.lastModified, + rec.expirationTime)); + + nsCacheEntry *entry = CreateCacheEntry(this, fullKey, rec); + + if (entry) + { + // make sure that the data file exists + nsOfflineCacheBinding *binding = (nsOfflineCacheBinding*)entry->Data(); + bool isFile; + rv = binding->mDataFile->IsFile(&isFile); + if (NS_FAILED(rv) || !isFile) + { + DeleteEntry(entry, false); + delete entry; + return nullptr; + } + + // lock the entry + Lock(*fullKey); + } + + return entry; +} + +nsresult +nsOfflineCacheDevice::DeactivateEntry(nsCacheEntry *entry) +{ + LOG(("nsOfflineCacheDevice::DeactivateEntry [key=%s]\n", + entry->Key()->get())); + + // This method is called to inform us that the nsCacheEntry object is going + // away. We should persist anything that needs to be persisted, or if the + // entry is doomed, we can go ahead and clear its storage. + + if (entry->IsDoomed()) + { + // remove corresponding row and file if they exist + + // the row should have been removed in DoomEntry... we could assert that + // that happened. otherwise, all we have to do here is delete the file + // on disk. + DeleteData(entry); + } + else if (((nsOfflineCacheBinding *)entry->Data())->IsNewEntry()) + { + // UPDATE the database row + + // Only new entries are updated, since offline cache is updated in + // transactions. New entries are those who is returned from + // BindEntry(). + + LOG(("nsOfflineCacheDevice::DeactivateEntry updating new entry\n")); + UpdateEntry(entry); + } else { + LOG(("nsOfflineCacheDevice::DeactivateEntry " + "skipping update since entry is not dirty\n")); + } + + // Unlock the entry + Unlock(*entry->Key()); + + delete entry; + + return NS_OK; +} + +nsresult +nsOfflineCacheDevice::BindEntry(nsCacheEntry *entry) +{ + NS_ENSURE_TRUE(Initialized(), NS_ERROR_NOT_INITIALIZED); + + LOG(("nsOfflineCacheDevice::BindEntry [key=%s]\n", entry->Key()->get())); + + NS_ENSURE_STATE(!entry->Data()); + + // This method is called to inform us that we have a new entry. The entry + // may collide with an existing entry in our DB, but if that happens we can + // assume that the entry is not being used. + + // INSERT the database row + + // XXX Assumption: if the row already exists, then FindEntry would have + // returned it. if that entry was doomed, then DoomEntry would have removed + // it from the table. so, we should always have to insert at this point. + + // Decompose the key into "ClientID" and "Key" + nsAutoCString keyBuf; + const char *cid, *key; + if (!DecomposeCacheEntryKey(entry->Key(), &cid, &key, keyBuf)) + return NS_ERROR_UNEXPECTED; + + // create binding, pick best generation number + RefPtr<nsOfflineCacheBinding> binding = + nsOfflineCacheBinding::Create(mCacheDirectory, entry->Key(), -1); + if (!binding) + return NS_ERROR_OUT_OF_MEMORY; + binding->MarkNewEntry(); + + nsOfflineCacheRecord rec; + rec.clientID = cid; + rec.key = key; + rec.metaData = nullptr; // don't write any metadata now. + rec.metaDataLen = 0; + rec.generation = binding->mGeneration; + rec.dataSize = 0; + rec.fetchCount = entry->FetchCount(); + rec.lastFetched = PRTimeFromSeconds(entry->LastFetched()); + rec.lastModified = PRTimeFromSeconds(entry->LastModified()); + rec.expirationTime = PRTimeFromSeconds(entry->ExpirationTime()); + + AutoResetStatement statement(mStatement_BindEntry); + + nsresult rv = statement->BindUTF8StringByIndex(0, nsDependentCString(rec.clientID)); + nsresult tmp = statement->BindUTF8StringByIndex(1, nsDependentCString(rec.key)); + if (NS_FAILED(tmp)) { + rv = tmp; + } + tmp = statement->BindBlobByIndex(2, rec.metaData, rec.metaDataLen); + if (NS_FAILED(tmp)) { + rv = tmp; + } + tmp = statement->BindInt32ByIndex(3, rec.generation); + if (NS_FAILED(tmp)) { + rv = tmp; + } + tmp = statement->BindInt32ByIndex(4, rec.dataSize); + if (NS_FAILED(tmp)) { + rv = tmp; + } + tmp = statement->BindInt32ByIndex(5, rec.fetchCount); + if (NS_FAILED(tmp)) { + rv = tmp; + } + tmp = statement->BindInt64ByIndex(6, rec.lastFetched); + if (NS_FAILED(tmp)) { + rv = tmp; + } + tmp = statement->BindInt64ByIndex(7, rec.lastModified); + if (NS_FAILED(tmp)) { + rv = tmp; + } + tmp = statement->BindInt64ByIndex(8, rec.expirationTime); + if (NS_FAILED(tmp)) { + rv = tmp; + } + NS_ENSURE_SUCCESS(rv, rv); + + bool hasRows; + rv = statement->ExecuteStep(&hasRows); + NS_ENSURE_SUCCESS(rv, rv); + NS_ASSERTION(!hasRows, "INSERT should not result in output"); + + entry->SetData(binding); + + // lock the entry + Lock(*entry->Key()); + + return NS_OK; +} + +void +nsOfflineCacheDevice::DoomEntry(nsCacheEntry *entry) +{ + LOG(("nsOfflineCacheDevice::DoomEntry [key=%s]\n", entry->Key()->get())); + + // This method is called to inform us that we should mark the entry to be + // deleted when it is no longer in use. + + // We can go ahead and delete the corresponding row in our table, + // but we must not delete the file on disk until we are deactivated. + // In another word, the file should be deleted if the entry had been + // deactivated. + + DeleteEntry(entry, !entry->IsActive()); +} + +nsresult +nsOfflineCacheDevice::OpenInputStreamForEntry(nsCacheEntry *entry, + nsCacheAccessMode mode, + uint32_t offset, + nsIInputStream **result) +{ + LOG(("nsOfflineCacheDevice::OpenInputStreamForEntry [key=%s]\n", + entry->Key()->get())); + + *result = nullptr; + + NS_ENSURE_TRUE(!offset || (offset < entry->DataSize()), NS_ERROR_INVALID_ARG); + + // return an input stream to the entry's data file. the stream + // may be read on a background thread. + + nsOfflineCacheBinding *binding = (nsOfflineCacheBinding *) entry->Data(); + NS_ENSURE_STATE(binding); + + nsCOMPtr<nsIInputStream> in; + NS_NewLocalFileInputStream(getter_AddRefs(in), binding->mDataFile, PR_RDONLY); + if (!in) + return NS_ERROR_UNEXPECTED; + + // respect |offset| param + if (offset != 0) + { + nsCOMPtr<nsISeekableStream> seekable = do_QueryInterface(in); + NS_ENSURE_TRUE(seekable, NS_ERROR_UNEXPECTED); + + seekable->Seek(nsISeekableStream::NS_SEEK_SET, offset); + } + + in.swap(*result); + return NS_OK; +} + +nsresult +nsOfflineCacheDevice::OpenOutputStreamForEntry(nsCacheEntry *entry, + nsCacheAccessMode mode, + uint32_t offset, + nsIOutputStream **result) +{ + LOG(("nsOfflineCacheDevice::OpenOutputStreamForEntry [key=%s]\n", + entry->Key()->get())); + + *result = nullptr; + + NS_ENSURE_TRUE(offset <= entry->DataSize(), NS_ERROR_INVALID_ARG); + + // return an output stream to the entry's data file. we can assume + // that the output stream will only be used on the main thread. + + nsOfflineCacheBinding *binding = (nsOfflineCacheBinding *) entry->Data(); + NS_ENSURE_STATE(binding); + + nsCOMPtr<nsIOutputStream> out; + NS_NewLocalFileOutputStream(getter_AddRefs(out), binding->mDataFile, + PR_WRONLY | PR_CREATE_FILE | PR_TRUNCATE, + 00600); + if (!out) + return NS_ERROR_UNEXPECTED; + + // respect |offset| param + nsCOMPtr<nsISeekableStream> seekable = do_QueryInterface(out); + NS_ENSURE_TRUE(seekable, NS_ERROR_UNEXPECTED); + if (offset != 0) + seekable->Seek(nsISeekableStream::NS_SEEK_SET, offset); + + // truncate the file at the given offset + seekable->SetEOF(); + + nsCOMPtr<nsIOutputStream> bufferedOut; + nsresult rv = + NS_NewBufferedOutputStream(getter_AddRefs(bufferedOut), out, 16 * 1024); + NS_ENSURE_SUCCESS(rv, rv); + + bufferedOut.swap(*result); + return NS_OK; +} + +nsresult +nsOfflineCacheDevice::GetFileForEntry(nsCacheEntry *entry, nsIFile **result) +{ + LOG(("nsOfflineCacheDevice::GetFileForEntry [key=%s]\n", + entry->Key()->get())); + + nsOfflineCacheBinding *binding = (nsOfflineCacheBinding *) entry->Data(); + NS_ENSURE_STATE(binding); + + NS_IF_ADDREF(*result = binding->mDataFile); + return NS_OK; +} + +nsresult +nsOfflineCacheDevice::OnDataSizeChange(nsCacheEntry *entry, int32_t deltaSize) +{ + LOG(("nsOfflineCacheDevice::OnDataSizeChange [key=%s delta=%d]\n", + entry->Key()->get(), deltaSize)); + + const int32_t DELTA_THRESHOLD = 1<<14; // 16k + + // called to notify us of an impending change in the total size of the + // specified entry. + + uint32_t oldSize = entry->DataSize(); + NS_ASSERTION(deltaSize >= 0 || int32_t(oldSize) + deltaSize >= 0, "oops"); + uint32_t newSize = int32_t(oldSize) + deltaSize; + UpdateEntrySize(entry, newSize); + + mDeltaCounter += deltaSize; // this may go negative + + if (mDeltaCounter >= DELTA_THRESHOLD) + { + if (CacheSize() > mCacheCapacity) { + // the entry will overrun the cache capacity, doom the entry + // and abort +#ifdef DEBUG + nsresult rv = +#endif + nsCacheService::DoomEntry(entry); + NS_ASSERTION(NS_SUCCEEDED(rv), "DoomEntry() failed."); + return NS_ERROR_ABORT; + } + + mDeltaCounter = 0; // reset counter + } + + return NS_OK; +} + +nsresult +nsOfflineCacheDevice::Visit(nsICacheVisitor *visitor) +{ + NS_ENSURE_TRUE(Initialized(), NS_ERROR_NOT_INITIALIZED); + + // called to enumerate the offline cache. + + nsCOMPtr<nsICacheDeviceInfo> deviceInfo = + new nsOfflineCacheDeviceInfo(this); + + bool keepGoing; + nsresult rv = visitor->VisitDevice(OFFLINE_CACHE_DEVICE_ID, deviceInfo, + &keepGoing); + if (NS_FAILED(rv)) + return rv; + + if (!keepGoing) + return NS_OK; + + // SELECT * from moz_cache; + + nsOfflineCacheRecord rec; + RefPtr<nsOfflineCacheEntryInfo> info = new nsOfflineCacheEntryInfo; + if (!info) + return NS_ERROR_OUT_OF_MEMORY; + info->mRec = &rec; + + // XXX may want to list columns explicitly + nsCOMPtr<mozIStorageStatement> statement; + rv = mDB->CreateStatement( + NS_LITERAL_CSTRING("SELECT * FROM moz_cache;"), + getter_AddRefs(statement)); + NS_ENSURE_SUCCESS(rv, rv); + + bool hasRows; + for (;;) + { + rv = statement->ExecuteStep(&hasRows); + if (NS_FAILED(rv) || !hasRows) + break; + + statement->GetSharedUTF8String(0, nullptr, &rec.clientID); + statement->GetSharedUTF8String(1, nullptr, &rec.key); + statement->GetSharedBlob(2, &rec.metaDataLen, + (const uint8_t **) &rec.metaData); + rec.generation = statement->AsInt32(3); + rec.dataSize = statement->AsInt32(4); + rec.fetchCount = statement->AsInt32(5); + rec.lastFetched = statement->AsInt64(6); + rec.lastModified = statement->AsInt64(7); + rec.expirationTime = statement->AsInt64(8); + + bool keepGoing; + rv = visitor->VisitEntry(OFFLINE_CACHE_DEVICE_ID, info, &keepGoing); + if (NS_FAILED(rv) || !keepGoing) + break; + } + + info->mRec = nullptr; + return NS_OK; +} + +nsresult +nsOfflineCacheDevice::EvictEntries(const char *clientID) +{ + NS_ENSURE_TRUE(Initialized(), NS_ERROR_NOT_INITIALIZED); + + LOG(("nsOfflineCacheDevice::EvictEntries [cid=%s]\n", + clientID ? clientID : "")); + + // called to evict all entries matching the given clientID. + + // need trigger to fire user defined function after a row is deleted + // so we can delete the corresponding data file. + EvictionObserver evictionObserver(mDB, mEvictionFunction); + + nsCOMPtr<mozIStorageStatement> statement; + nsresult rv; + if (clientID) + { + rv = mDB->CreateStatement(NS_LITERAL_CSTRING("DELETE FROM moz_cache WHERE ClientID=?;"), + getter_AddRefs(statement)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = statement->BindUTF8StringByIndex(0, nsDependentCString(clientID)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = statement->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + + rv = mDB->CreateStatement(NS_LITERAL_CSTRING("DELETE FROM moz_cache_groups WHERE ActiveClientID=?;"), + getter_AddRefs(statement)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = statement->BindUTF8StringByIndex(0, nsDependentCString(clientID)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = statement->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + + // TODO - Should update internal hashtables. + // Low priority, since this API is not widely used. + } + else + { + rv = mDB->CreateStatement(NS_LITERAL_CSTRING("DELETE FROM moz_cache;"), + getter_AddRefs(statement)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = statement->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + + rv = mDB->CreateStatement(NS_LITERAL_CSTRING("DELETE FROM moz_cache_groups;"), + getter_AddRefs(statement)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = statement->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + + MutexAutoLock lock(mLock); + mCaches.Clear(); + mActiveCaches.Clear(); + mActiveCachesByGroup.Clear(); + } + + evictionObserver.Apply(); + + statement = nullptr; + // Also evict any namespaces associated with this clientID. + if (clientID) + { + rv = mDB->CreateStatement(NS_LITERAL_CSTRING("DELETE FROM moz_cache_namespaces WHERE ClientID=?"), + getter_AddRefs(statement)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = statement->BindUTF8StringByIndex(0, nsDependentCString(clientID)); + NS_ENSURE_SUCCESS(rv, rv); + } + else + { + rv = mDB->CreateStatement(NS_LITERAL_CSTRING("DELETE FROM moz_cache_namespaces;"), + getter_AddRefs(statement)); + NS_ENSURE_SUCCESS(rv, rv); + } + + rv = statement->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult +nsOfflineCacheDevice::MarkEntry(const nsCString &clientID, + const nsACString &key, + uint32_t typeBits) +{ + NS_ENSURE_TRUE(Initialized(), NS_ERROR_NOT_INITIALIZED); + + LOG(("nsOfflineCacheDevice::MarkEntry [cid=%s, key=%s, typeBits=%d]\n", + clientID.get(), PromiseFlatCString(key).get(), typeBits)); + + AutoResetStatement statement(mStatement_MarkEntry); + nsresult rv = statement->BindInt32ByIndex(0, typeBits); + NS_ENSURE_SUCCESS(rv, rv); + rv = statement->BindUTF8StringByIndex(1, clientID); + NS_ENSURE_SUCCESS(rv, rv); + rv = statement->BindUTF8StringByIndex(2, key); + NS_ENSURE_SUCCESS(rv, rv); + + rv = statement->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult +nsOfflineCacheDevice::UnmarkEntry(const nsCString &clientID, + const nsACString &key, + uint32_t typeBits) +{ + NS_ENSURE_TRUE(Initialized(), NS_ERROR_NOT_INITIALIZED); + + LOG(("nsOfflineCacheDevice::UnmarkEntry [cid=%s, key=%s, typeBits=%d]\n", + clientID.get(), PromiseFlatCString(key).get(), typeBits)); + + AutoResetStatement statement(mStatement_UnmarkEntry); + nsresult rv = statement->BindInt32ByIndex(0, typeBits); + NS_ENSURE_SUCCESS(rv, rv); + rv = statement->BindUTF8StringByIndex(1, clientID); + NS_ENSURE_SUCCESS(rv, rv); + rv = statement->BindUTF8StringByIndex(2, key); + NS_ENSURE_SUCCESS(rv, rv); + + rv = statement->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + + // Remove the entry if it is now empty. + + EvictionObserver evictionObserver(mDB, mEvictionFunction); + + AutoResetStatement cleanupStatement(mStatement_CleanupUnmarked); + rv = cleanupStatement->BindUTF8StringByIndex(0, clientID); + NS_ENSURE_SUCCESS(rv, rv); + rv = cleanupStatement->BindUTF8StringByIndex(1, key); + NS_ENSURE_SUCCESS(rv, rv); + + rv = cleanupStatement->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + + evictionObserver.Apply(); + + return NS_OK; +} + +nsresult +nsOfflineCacheDevice::GetMatchingNamespace(const nsCString &clientID, + const nsACString &key, + nsIApplicationCacheNamespace **out) +{ + NS_ENSURE_TRUE(Initialized(), NS_ERROR_NOT_INITIALIZED); + + LOG(("nsOfflineCacheDevice::GetMatchingNamespace [cid=%s, key=%s]\n", + clientID.get(), PromiseFlatCString(key).get())); + + nsresult rv; + + AutoResetStatement statement(mStatement_FindNamespaceEntry); + + rv = statement->BindUTF8StringByIndex(0, clientID); + NS_ENSURE_SUCCESS(rv, rv); + rv = statement->BindUTF8StringByIndex(1, key); + NS_ENSURE_SUCCESS(rv, rv); + + bool hasRows; + rv = statement->ExecuteStep(&hasRows); + NS_ENSURE_SUCCESS(rv, rv); + + *out = nullptr; + + bool found = false; + nsCString nsSpec; + int32_t nsType = 0; + nsCString nsData; + + while (hasRows) + { + int32_t itemType; + rv = statement->GetInt32(2, &itemType); + NS_ENSURE_SUCCESS(rv, rv); + + if (!found || itemType > nsType) + { + nsType = itemType; + + rv = statement->GetUTF8String(0, nsSpec); + NS_ENSURE_SUCCESS(rv, rv); + + rv = statement->GetUTF8String(1, nsData); + NS_ENSURE_SUCCESS(rv, rv); + + found = true; + } + + rv = statement->ExecuteStep(&hasRows); + NS_ENSURE_SUCCESS(rv, rv); + } + + if (found) { + nsCOMPtr<nsIApplicationCacheNamespace> ns = + new nsApplicationCacheNamespace(); + if (!ns) + return NS_ERROR_OUT_OF_MEMORY; + rv = ns->Init(nsType, nsSpec, nsData); + NS_ENSURE_SUCCESS(rv, rv); + + ns.swap(*out); + } + + return NS_OK; +} + +nsresult +nsOfflineCacheDevice::CacheOpportunistically(const nsCString &clientID, + const nsACString &key) +{ + // XXX: We should also be propagating this cache entry to other matching + // caches. See bug 444807. + + return MarkEntry(clientID, key, nsIApplicationCache::ITEM_OPPORTUNISTIC); +} + +nsresult +nsOfflineCacheDevice::GetTypes(const nsCString &clientID, + const nsACString &key, + uint32_t *typeBits) +{ + NS_ENSURE_TRUE(Initialized(), NS_ERROR_NOT_INITIALIZED); + + LOG(("nsOfflineCacheDevice::GetTypes [cid=%s, key=%s]\n", + clientID.get(), PromiseFlatCString(key).get())); + + AutoResetStatement statement(mStatement_GetTypes); + nsresult rv = statement->BindUTF8StringByIndex(0, clientID); + NS_ENSURE_SUCCESS(rv, rv); + rv = statement->BindUTF8StringByIndex(1, key); + NS_ENSURE_SUCCESS(rv, rv); + + bool hasRows; + rv = statement->ExecuteStep(&hasRows); + NS_ENSURE_SUCCESS(rv, rv); + + if (!hasRows) + return NS_ERROR_CACHE_KEY_NOT_FOUND; + + *typeBits = statement->AsInt32(0); + + return NS_OK; +} + +nsresult +nsOfflineCacheDevice::GatherEntries(const nsCString &clientID, + uint32_t typeBits, + uint32_t *count, + char ***keys) +{ + NS_ENSURE_TRUE(Initialized(), NS_ERROR_NOT_INITIALIZED); + + LOG(("nsOfflineCacheDevice::GatherEntries [cid=%s, typeBits=%X]\n", + clientID.get(), typeBits)); + + AutoResetStatement statement(mStatement_GatherEntries); + nsresult rv = statement->BindUTF8StringByIndex(0, clientID); + NS_ENSURE_SUCCESS(rv, rv); + + rv = statement->BindInt32ByIndex(1, typeBits); + NS_ENSURE_SUCCESS(rv, rv); + + return RunSimpleQuery(mStatement_GatherEntries, 0, count, keys); +} + +nsresult +nsOfflineCacheDevice::AddNamespace(const nsCString &clientID, + nsIApplicationCacheNamespace *ns) +{ + NS_ENSURE_TRUE(Initialized(), NS_ERROR_NOT_INITIALIZED); + + nsCString namespaceSpec; + nsresult rv = ns->GetNamespaceSpec(namespaceSpec); + NS_ENSURE_SUCCESS(rv, rv); + + nsCString data; + rv = ns->GetData(data); + NS_ENSURE_SUCCESS(rv, rv); + + uint32_t itemType; + rv = ns->GetItemType(&itemType); + NS_ENSURE_SUCCESS(rv, rv); + + LOG(("nsOfflineCacheDevice::AddNamespace [cid=%s, ns=%s, data=%s, type=%d]", + clientID.get(), namespaceSpec.get(), data.get(), itemType)); + + AutoResetStatement statement(mStatement_InsertNamespaceEntry); + + rv = statement->BindUTF8StringByIndex(0, clientID); + NS_ENSURE_SUCCESS(rv, rv); + + rv = statement->BindUTF8StringByIndex(1, namespaceSpec); + NS_ENSURE_SUCCESS(rv, rv); + + rv = statement->BindUTF8StringByIndex(2, data); + NS_ENSURE_SUCCESS(rv, rv); + + rv = statement->BindInt32ByIndex(3, itemType); + NS_ENSURE_SUCCESS(rv, rv); + + rv = statement->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult +nsOfflineCacheDevice::GetUsage(const nsACString &clientID, + uint32_t *usage) +{ + NS_ENSURE_TRUE(Initialized(), NS_ERROR_NOT_INITIALIZED); + + LOG(("nsOfflineCacheDevice::GetUsage [cid=%s]\n", + PromiseFlatCString(clientID).get())); + + *usage = 0; + + AutoResetStatement statement(mStatement_ApplicationCacheSize); + + nsresult rv = statement->BindUTF8StringByIndex(0, clientID); + NS_ENSURE_SUCCESS(rv, rv); + + bool hasRows; + rv = statement->ExecuteStep(&hasRows); + NS_ENSURE_SUCCESS(rv, rv); + + if (!hasRows) + return NS_OK; + + *usage = static_cast<uint32_t>(statement->AsInt32(0)); + + return NS_OK; +} + +nsresult +nsOfflineCacheDevice::GetGroups(uint32_t *count, + char ***keys) +{ + NS_ENSURE_TRUE(Initialized(), NS_ERROR_NOT_INITIALIZED); + + LOG(("nsOfflineCacheDevice::GetGroups")); + + return RunSimpleQuery(mStatement_EnumerateGroups, 0, count, keys); +} + +nsresult +nsOfflineCacheDevice::GetGroupsTimeOrdered(uint32_t *count, + char ***keys) +{ + NS_ENSURE_TRUE(Initialized(), NS_ERROR_NOT_INITIALIZED); + + LOG(("nsOfflineCacheDevice::GetGroupsTimeOrder")); + + return RunSimpleQuery(mStatement_EnumerateGroupsTimeOrder, 0, count, keys); +} + +bool +nsOfflineCacheDevice::IsLocked(const nsACString &key) +{ + MutexAutoLock lock(mLock); + return mLockedEntries.GetEntry(key); +} + +void +nsOfflineCacheDevice::Lock(const nsACString &key) +{ + MutexAutoLock lock(mLock); + mLockedEntries.PutEntry(key); +} + +void +nsOfflineCacheDevice::Unlock(const nsACString &key) +{ + MutexAutoLock lock(mLock); + mLockedEntries.RemoveEntry(key); +} + +nsresult +nsOfflineCacheDevice::RunSimpleQuery(mozIStorageStatement * statement, + uint32_t resultIndex, + uint32_t * count, + char *** values) +{ + bool hasRows; + nsresult rv = statement->ExecuteStep(&hasRows); + NS_ENSURE_SUCCESS(rv, rv); + + nsTArray<nsCString> valArray; + while (hasRows) + { + uint32_t length; + valArray.AppendElement( + nsDependentCString(statement->AsSharedUTF8String(resultIndex, &length))); + + rv = statement->ExecuteStep(&hasRows); + NS_ENSURE_SUCCESS(rv, rv); + } + + *count = valArray.Length(); + char **ret = static_cast<char **>(moz_xmalloc(*count * sizeof(char*))); + if (!ret) return NS_ERROR_OUT_OF_MEMORY; + + for (uint32_t i = 0; i < *count; i++) { + ret[i] = NS_strdup(valArray[i].get()); + if (!ret[i]) { + NS_FREE_XPCOM_ALLOCATED_POINTER_ARRAY(i, ret); + return NS_ERROR_OUT_OF_MEMORY; + } + } + + *values = ret; + + return NS_OK; +} + +nsresult +nsOfflineCacheDevice::CreateApplicationCache(const nsACString &group, + nsIApplicationCache **out) +{ + *out = nullptr; + + nsCString clientID; + // Some characters are special in the clientID. Escape the groupID + // before putting it in to the client key. + if (!NS_Escape(nsCString(group), clientID, url_Path)) { + return NS_ERROR_OUT_OF_MEMORY; + } + + PRTime now = PR_Now(); + + // Include the timestamp to guarantee uniqueness across runs, and + // the gNextTemporaryClientID for uniqueness within a second. + clientID.Append(nsPrintfCString("|%016lld|%d", + now / PR_USEC_PER_SEC, + gNextTemporaryClientID++)); + + nsCOMPtr<nsIApplicationCache> cache = new nsApplicationCache(this, + group, + clientID); + if (!cache) + return NS_ERROR_OUT_OF_MEMORY; + + nsCOMPtr<nsIWeakReference> weak = do_GetWeakReference(cache); + if (!weak) + return NS_ERROR_OUT_OF_MEMORY; + + MutexAutoLock lock(mLock); + mCaches.Put(clientID, weak); + + cache.swap(*out); + + return NS_OK; +} + +nsresult +nsOfflineCacheDevice::GetApplicationCache(const nsACString &clientID, + nsIApplicationCache **out) +{ + MutexAutoLock lock(mLock); + return GetApplicationCache_Unlocked(clientID, out); +} + +nsresult +nsOfflineCacheDevice::GetApplicationCache_Unlocked(const nsACString &clientID, + nsIApplicationCache **out) +{ + *out = nullptr; + + nsCOMPtr<nsIApplicationCache> cache; + + nsWeakPtr weak; + if (mCaches.Get(clientID, getter_AddRefs(weak))) + cache = do_QueryReferent(weak); + + if (!cache) + { + nsCString group; + nsresult rv = GetGroupForCache(clientID, group); + NS_ENSURE_SUCCESS(rv, rv); + + if (group.IsEmpty()) { + return NS_OK; + } + + cache = new nsApplicationCache(this, group, clientID); + weak = do_GetWeakReference(cache); + if (!weak) + return NS_ERROR_OUT_OF_MEMORY; + + mCaches.Put(clientID, weak); + } + + cache.swap(*out); + + return NS_OK; +} + +nsresult +nsOfflineCacheDevice::GetActiveCache(const nsACString &group, + nsIApplicationCache **out) +{ + *out = nullptr; + + MutexAutoLock lock(mLock); + + nsCString *clientID; + if (mActiveCachesByGroup.Get(group, &clientID)) + return GetApplicationCache_Unlocked(*clientID, out); + + return NS_OK; +} + +nsresult +nsOfflineCacheDevice::DeactivateGroup(const nsACString &group) +{ + NS_ENSURE_TRUE(Initialized(), NS_ERROR_NOT_INITIALIZED); + + nsCString *active = nullptr; + + AutoResetStatement statement(mStatement_DeactivateGroup); + nsresult rv = statement->BindUTF8StringByIndex(0, group); + NS_ENSURE_SUCCESS(rv, rv); + + rv = statement->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + + MutexAutoLock lock(mLock); + + if (mActiveCachesByGroup.Get(group, &active)) + { + mActiveCaches.RemoveEntry(*active); + mActiveCachesByGroup.Remove(group); + active = nullptr; + } + + return NS_OK; +} + +nsresult +nsOfflineCacheDevice::Evict(nsILoadContextInfo *aInfo) +{ + NS_ENSURE_TRUE(Initialized(), NS_ERROR_NOT_INITIALIZED); + + NS_ENSURE_ARG(aInfo); + + nsresult rv; + + mozilla::OriginAttributes const *oa = aInfo->OriginAttributesPtr(); + + if (oa->mAppId == NECKO_NO_APP_ID && oa->mInIsolatedMozBrowser == false) { + nsCOMPtr<nsICacheService> serv = do_GetService(kCacheServiceCID, &rv); + NS_ENSURE_SUCCESS(rv, rv); + + return nsCacheService::GlobalInstance()->EvictEntriesInternal(nsICache::STORE_OFFLINE); + } + + nsAutoCString jaridsuffix; + jaridsuffix.Append('%'); + + nsAutoCString suffix; + oa->CreateSuffix(suffix); + jaridsuffix.Append('#'); + jaridsuffix.Append(suffix); + + AutoResetStatement statement(mStatement_EnumerateApps); + rv = statement->BindUTF8StringByIndex(0, jaridsuffix); + NS_ENSURE_SUCCESS(rv, rv); + + bool hasRows; + rv = statement->ExecuteStep(&hasRows); + NS_ENSURE_SUCCESS(rv, rv); + + while (hasRows) { + nsAutoCString group; + rv = statement->GetUTF8String(0, group); + NS_ENSURE_SUCCESS(rv, rv); + + nsCString clientID; + rv = statement->GetUTF8String(1, clientID); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr<nsIRunnable> ev = + new nsOfflineCacheDiscardCache(this, group, clientID); + + rv = nsCacheService::DispatchToCacheIOThread(ev); + NS_ENSURE_SUCCESS(rv, rv); + + rv = statement->ExecuteStep(&hasRows); + NS_ENSURE_SUCCESS(rv, rv); + } + + return NS_OK; +} + +namespace { // anon + +class OriginMatch final : public mozIStorageFunction +{ + ~OriginMatch() {} + mozilla::OriginAttributesPattern const mPattern; + + NS_DECL_ISUPPORTS + NS_DECL_MOZISTORAGEFUNCTION + explicit OriginMatch(mozilla::OriginAttributesPattern const &aPattern) + : mPattern(aPattern) {} +}; + +NS_IMPL_ISUPPORTS(OriginMatch, mozIStorageFunction) + +NS_IMETHODIMP +OriginMatch::OnFunctionCall(mozIStorageValueArray* aFunctionArguments, nsIVariant** aResult) +{ + nsresult rv; + + nsAutoCString groupId; + rv = aFunctionArguments->GetUTF8String(0, groupId); + NS_ENSURE_SUCCESS(rv, rv); + + int32_t hash = groupId.Find(NS_LITERAL_CSTRING("#")); + if (hash == kNotFound) { + // Just ignore... + return NS_OK; + } + + ++hash; + + nsDependentCSubstring suffix(groupId.BeginReading() + hash, groupId.Length() - hash); + + mozilla::NeckoOriginAttributes oa; + bool ok = oa.PopulateFromSuffix(suffix); + NS_ENSURE_TRUE(ok, NS_ERROR_UNEXPECTED); + + bool match = mPattern.Matches(oa); + + RefPtr<nsVariant> outVar(new nsVariant()); + rv = outVar->SetAsUint32(match ? 1 : 0); + NS_ENSURE_SUCCESS(rv, rv); + + outVar.forget(aResult); + return NS_OK; +} + +} // anon + +nsresult +nsOfflineCacheDevice::Evict(mozilla::OriginAttributesPattern const &aPattern) +{ + NS_ENSURE_TRUE(Initialized(), NS_ERROR_NOT_INITIALIZED); + + nsresult rv; + + nsCOMPtr<mozIStorageFunction> function1(new OriginMatch(aPattern)); + rv = mDB->CreateFunction(NS_LITERAL_CSTRING("ORIGIN_MATCH"), 1, function1); + NS_ENSURE_SUCCESS(rv, rv); + + class AutoRemoveFunc { + public: + mozIStorageConnection* mDB; + explicit AutoRemoveFunc(mozIStorageConnection* aDB) : mDB(aDB) {} + ~AutoRemoveFunc() { + mDB->RemoveFunction(NS_LITERAL_CSTRING("ORIGIN_MATCH")); + } + }; + AutoRemoveFunc autoRemove(mDB); + + nsCOMPtr<mozIStorageStatement> statement; + rv = mDB->CreateStatement( + NS_LITERAL_CSTRING("SELECT GroupID, ActiveClientID FROM moz_cache_groups WHERE ORIGIN_MATCH(GroupID);"), + getter_AddRefs(statement)); + NS_ENSURE_SUCCESS(rv, rv); + + AutoResetStatement statementScope(statement); + + bool hasRows; + rv = statement->ExecuteStep(&hasRows); + NS_ENSURE_SUCCESS(rv, rv); + + while (hasRows) { + nsAutoCString group; + rv = statement->GetUTF8String(0, group); + NS_ENSURE_SUCCESS(rv, rv); + + nsCString clientID; + rv = statement->GetUTF8String(1, clientID); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr<nsIRunnable> ev = + new nsOfflineCacheDiscardCache(this, group, clientID); + + rv = nsCacheService::DispatchToCacheIOThread(ev); + NS_ENSURE_SUCCESS(rv, rv); + + rv = statement->ExecuteStep(&hasRows); + NS_ENSURE_SUCCESS(rv, rv); + } + + return NS_OK; +} + +bool +nsOfflineCacheDevice::CanUseCache(nsIURI *keyURI, + const nsACString &clientID, + nsILoadContextInfo *loadContextInfo) +{ + { + MutexAutoLock lock(mLock); + if (!mActiveCaches.Contains(clientID)) + return false; + } + + nsAutoCString groupID; + nsresult rv = GetGroupForCache(clientID, groupID); + NS_ENSURE_SUCCESS(rv, false); + + nsCOMPtr<nsIURI> groupURI; + rv = NS_NewURI(getter_AddRefs(groupURI), groupID); + if (NS_FAILED(rv)) { + return false; + } + + // When we are choosing an initial cache to load the top + // level document from, the URL of that document must have + // the same origin as the manifest, according to the spec. + // The following check is here because explicit, fallback + // and dynamic entries might have origin different from the + // manifest origin. + if (!NS_SecurityCompareURIs(keyURI, groupURI, + GetStrictFileOriginPolicy())) { + return false; + } + + // Check the groupID we found is equal to groupID based + // on the load context demanding load from app cache. + // This is check of extended origin. + + nsAutoCString originSuffix; + loadContextInfo->OriginAttributesPtr()->CreateSuffix(originSuffix); + + nsAutoCString demandedGroupID; + rv = BuildApplicationCacheGroupID(groupURI, originSuffix, demandedGroupID); + NS_ENSURE_SUCCESS(rv, false); + + if (groupID != demandedGroupID) { + return false; + } + + return true; +} + + +nsresult +nsOfflineCacheDevice::ChooseApplicationCache(const nsACString &key, + nsILoadContextInfo *loadContextInfo, + nsIApplicationCache **out) +{ + NS_ENSURE_TRUE(Initialized(), NS_ERROR_NOT_INITIALIZED); + + NS_ENSURE_ARG(loadContextInfo); + + nsresult rv; + + *out = nullptr; + + nsCOMPtr<nsIURI> keyURI; + rv = NS_NewURI(getter_AddRefs(keyURI), key); + NS_ENSURE_SUCCESS(rv, rv); + + // First try to find a matching cache entry. + AutoResetStatement statement(mStatement_FindClient); + rv = statement->BindUTF8StringByIndex(0, key); + NS_ENSURE_SUCCESS(rv, rv); + + bool hasRows; + rv = statement->ExecuteStep(&hasRows); + NS_ENSURE_SUCCESS(rv, rv); + + while (hasRows) { + int32_t itemType; + rv = statement->GetInt32(1, &itemType); + NS_ENSURE_SUCCESS(rv, rv); + + if (!(itemType & nsIApplicationCache::ITEM_FOREIGN)) { + nsAutoCString clientID; + rv = statement->GetUTF8String(0, clientID); + NS_ENSURE_SUCCESS(rv, rv); + + if (CanUseCache(keyURI, clientID, loadContextInfo)) { + return GetApplicationCache(clientID, out); + } + } + + rv = statement->ExecuteStep(&hasRows); + NS_ENSURE_SUCCESS(rv, rv); + } + + // OK, we didn't find an exact match. Search for a client with a + // matching namespace. + + AutoResetStatement nsstatement(mStatement_FindClientByNamespace); + + rv = nsstatement->BindUTF8StringByIndex(0, key); + NS_ENSURE_SUCCESS(rv, rv); + + rv = nsstatement->ExecuteStep(&hasRows); + NS_ENSURE_SUCCESS(rv, rv); + + while (hasRows) + { + int32_t itemType; + rv = nsstatement->GetInt32(1, &itemType); + NS_ENSURE_SUCCESS(rv, rv); + + // Don't associate with a cache based solely on a whitelist entry + if (!(itemType & nsIApplicationCacheNamespace::NAMESPACE_BYPASS)) { + nsAutoCString clientID; + rv = nsstatement->GetUTF8String(0, clientID); + NS_ENSURE_SUCCESS(rv, rv); + + if (CanUseCache(keyURI, clientID, loadContextInfo)) { + return GetApplicationCache(clientID, out); + } + } + + rv = nsstatement->ExecuteStep(&hasRows); + NS_ENSURE_SUCCESS(rv, rv); + } + + return NS_OK; +} + +nsresult +nsOfflineCacheDevice::CacheOpportunistically(nsIApplicationCache* cache, + const nsACString &key) +{ + NS_ENSURE_ARG_POINTER(cache); + + nsresult rv; + + nsAutoCString clientID; + rv = cache->GetClientID(clientID); + NS_ENSURE_SUCCESS(rv, rv); + + return CacheOpportunistically(clientID, key); +} + +nsresult +nsOfflineCacheDevice::ActivateCache(const nsCSubstring &group, + const nsCSubstring &clientID) +{ + NS_ENSURE_TRUE(Initialized(), NS_ERROR_NOT_INITIALIZED); + + AutoResetStatement statement(mStatement_ActivateClient); + nsresult rv = statement->BindUTF8StringByIndex(0, group); + NS_ENSURE_SUCCESS(rv, rv); + rv = statement->BindUTF8StringByIndex(1, clientID); + NS_ENSURE_SUCCESS(rv, rv); + rv = statement->BindInt32ByIndex(2, SecondsFromPRTime(PR_Now())); + NS_ENSURE_SUCCESS(rv, rv); + + rv = statement->Execute(); + NS_ENSURE_SUCCESS(rv, rv); + + MutexAutoLock lock(mLock); + + nsCString *active; + if (mActiveCachesByGroup.Get(group, &active)) + { + mActiveCaches.RemoveEntry(*active); + mActiveCachesByGroup.Remove(group); + active = nullptr; + } + + if (!clientID.IsEmpty()) + { + mActiveCaches.PutEntry(clientID); + mActiveCachesByGroup.Put(group, new nsCString(clientID)); + } + + return NS_OK; +} + +bool +nsOfflineCacheDevice::IsActiveCache(const nsCSubstring &group, + const nsCSubstring &clientID) +{ + nsCString *active = nullptr; + MutexAutoLock lock(mLock); + return mActiveCachesByGroup.Get(group, &active) && *active == clientID; +} + +/** + * Preference accessors + */ + +void +nsOfflineCacheDevice::SetCacheParentDirectory(nsIFile *parentDir) +{ + if (Initialized()) + { + NS_ERROR("cannot switch cache directory once initialized"); + return; + } + + if (!parentDir) + { + mCacheDirectory = nullptr; + return; + } + + // ensure parent directory exists + nsresult rv = EnsureDir(parentDir); + if (NS_FAILED(rv)) + { + NS_WARNING("unable to create parent directory"); + return; + } + + mBaseDirectory = parentDir; + + // cache dir may not exist, but that's ok + nsCOMPtr<nsIFile> dir; + rv = parentDir->Clone(getter_AddRefs(dir)); + if (NS_FAILED(rv)) + return; + rv = dir->AppendNative(NS_LITERAL_CSTRING("OfflineCache")); + if (NS_FAILED(rv)) + return; + + mCacheDirectory = do_QueryInterface(dir); +} + +void +nsOfflineCacheDevice::SetCapacity(uint32_t capacity) +{ + mCacheCapacity = capacity * 1024; +} + +bool +nsOfflineCacheDevice::AutoShutdown(nsIApplicationCache * aAppCache) +{ + if (!mAutoShutdown) + return false; + + mAutoShutdown = false; + + Shutdown(); + + nsCOMPtr<nsICacheService> serv = do_GetService(kCacheServiceCID); + RefPtr<nsCacheService> cacheService = nsCacheService::GlobalInstance(); + cacheService->RemoveCustomOfflineDevice(this); + + nsAutoCString clientID; + aAppCache->GetClientID(clientID); + + MutexAutoLock lock(mLock); + mCaches.Remove(clientID); + + return true; +} diff --git a/netwerk/cache/nsDiskCacheDeviceSQL.h b/netwerk/cache/nsDiskCacheDeviceSQL.h new file mode 100644 index 000000000..fcde58d3d --- /dev/null +++ b/netwerk/cache/nsDiskCacheDeviceSQL.h @@ -0,0 +1,290 @@ +/* vim:set ts=2 sw=2 sts=2 et cin: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef nsOfflineCacheDevice_h__ +#define nsOfflineCacheDevice_h__ + +#include "nsCacheDevice.h" +#include "nsIApplicationCache.h" +#include "nsIApplicationCacheService.h" +#include "nsIObserver.h" +#include "mozIStorageConnection.h" +#include "mozIStorageFunction.h" +#include "nsIFile.h" +#include "nsAutoPtr.h" +#include "nsCOMPtr.h" +#include "nsCOMArray.h" +#include "nsInterfaceHashtable.h" +#include "nsClassHashtable.h" +#include "nsWeakReference.h" +#include "mozilla/Attributes.h" +#include "mozilla/Mutex.h" + +class nsIURI; +class nsOfflineCacheDevice; +class mozIStorageService; +class nsILoadContextInfo; +namespace mozilla { class OriginAttributesPattern; } + +class nsApplicationCacheNamespace final : public nsIApplicationCacheNamespace +{ +public: + NS_DECL_ISUPPORTS + NS_DECL_NSIAPPLICATIONCACHENAMESPACE + + nsApplicationCacheNamespace() : mItemType(0) {} + +private: + ~nsApplicationCacheNamespace() {} + + uint32_t mItemType; + nsCString mNamespaceSpec; + nsCString mData; +}; + +class nsOfflineCacheEvictionFunction final : public mozIStorageFunction { +public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_MOZISTORAGEFUNCTION + + explicit nsOfflineCacheEvictionFunction(nsOfflineCacheDevice *device); + + void Init(); + void Reset(); + void Apply(); + +private: + ~nsOfflineCacheEvictionFunction() {} + + nsOfflineCacheDevice *mDevice; + bool mTLSInited; +}; + +class nsOfflineCacheDevice final : public nsCacheDevice + , public nsISupports +{ +public: + nsOfflineCacheDevice(); + + NS_DECL_THREADSAFE_ISUPPORTS + + /** + * nsCacheDevice methods + */ + + virtual nsresult Init() override; + nsresult InitWithSqlite(mozIStorageService * ss); + virtual nsresult Shutdown() override; + + virtual const char * GetDeviceID(void) override; + virtual nsCacheEntry * FindEntry(nsCString * key, bool *collision) override; + virtual nsresult DeactivateEntry(nsCacheEntry * entry) override; + virtual nsresult BindEntry(nsCacheEntry * entry) override; + virtual void DoomEntry( nsCacheEntry * entry ) override; + + virtual nsresult OpenInputStreamForEntry(nsCacheEntry * entry, + nsCacheAccessMode mode, + uint32_t offset, + nsIInputStream ** result) override; + + virtual nsresult OpenOutputStreamForEntry(nsCacheEntry * entry, + nsCacheAccessMode mode, + uint32_t offset, + nsIOutputStream ** result) override; + + virtual nsresult GetFileForEntry(nsCacheEntry * entry, + nsIFile ** result) override; + + virtual nsresult OnDataSizeChange(nsCacheEntry * entry, int32_t deltaSize) override; + + virtual nsresult Visit(nsICacheVisitor * visitor) override; + + virtual nsresult EvictEntries(const char * clientID) override; + + /* Entry ownership */ + nsresult GetOwnerDomains(const char * clientID, + uint32_t * count, + char *** domains); + nsresult GetOwnerURIs(const char * clientID, + const nsACString & ownerDomain, + uint32_t * count, + char *** uris); + nsresult SetOwnedKeys(const char * clientID, + const nsACString & ownerDomain, + const nsACString & ownerUrl, + uint32_t count, + const char ** keys); + nsresult GetOwnedKeys(const char * clientID, + const nsACString & ownerDomain, + const nsACString & ownerUrl, + uint32_t * count, + char *** keys); + nsresult AddOwnedKey(const char * clientID, + const nsACString & ownerDomain, + const nsACString & ownerURI, + const nsACString & key); + nsresult RemoveOwnedKey(const char * clientID, + const nsACString & ownerDomain, + const nsACString & ownerURI, + const nsACString & key); + nsresult KeyIsOwned(const char * clientID, + const nsACString & ownerDomain, + const nsACString & ownerURI, + const nsACString & key, + bool * isOwned); + + nsresult ClearKeysOwnedByDomain(const char *clientID, + const nsACString &ownerDomain); + nsresult EvictUnownedEntries(const char *clientID); + + static nsresult BuildApplicationCacheGroupID(nsIURI *aManifestURL, + nsACString const &aOriginSuffix, + nsACString &_result); + + nsresult ActivateCache(const nsCSubstring &group, + const nsCSubstring &clientID); + bool IsActiveCache(const nsCSubstring &group, + const nsCSubstring &clientID); + nsresult CreateApplicationCache(const nsACString &group, + nsIApplicationCache **out); + + nsresult GetApplicationCache(const nsACString &clientID, + nsIApplicationCache **out); + nsresult GetApplicationCache_Unlocked(const nsACString &clientID, + nsIApplicationCache **out); + + nsresult GetActiveCache(const nsACString &group, + nsIApplicationCache **out); + + nsresult DeactivateGroup(const nsACString &group); + + nsresult ChooseApplicationCache(const nsACString &key, + nsILoadContextInfo *loadContext, + nsIApplicationCache **out); + + nsresult CacheOpportunistically(nsIApplicationCache* cache, + const nsACString &key); + + nsresult Evict(nsILoadContextInfo *aInfo); + nsresult Evict(mozilla::OriginAttributesPattern const &aPattern); + + nsresult GetGroups(uint32_t *count,char ***keys); + + nsresult GetGroupsTimeOrdered(uint32_t *count, + char ***keys); + + bool IsLocked(const nsACString &key); + void Lock(const nsACString &key); + void Unlock(const nsACString &key); + + /** + * Preference accessors + */ + + void SetCacheParentDirectory(nsIFile * parentDir); + void SetCapacity(uint32_t capacity); + void SetAutoShutdown() { mAutoShutdown = true; } + bool AutoShutdown(nsIApplicationCache * aAppCache); + + nsIFile * BaseDirectory() { return mBaseDirectory; } + nsIFile * CacheDirectory() { return mCacheDirectory; } + uint32_t CacheCapacity() { return mCacheCapacity; } + uint32_t CacheSize(); + uint32_t EntryCount(); + +private: + ~nsOfflineCacheDevice(); + + friend class nsApplicationCache; + + static bool GetStrictFileOriginPolicy(); + + bool Initialized() { return mDB != nullptr; } + + nsresult InitActiveCaches(); + nsresult UpdateEntry(nsCacheEntry *entry); + nsresult UpdateEntrySize(nsCacheEntry *entry, uint32_t newSize); + nsresult DeleteEntry(nsCacheEntry *entry, bool deleteData); + nsresult DeleteData(nsCacheEntry *entry); + nsresult EnableEvictionObserver(); + nsresult DisableEvictionObserver(); + + bool CanUseCache(nsIURI *keyURI, const nsACString &clientID, nsILoadContextInfo *loadContext); + + nsresult MarkEntry(const nsCString &clientID, + const nsACString &key, + uint32_t typeBits); + nsresult UnmarkEntry(const nsCString &clientID, + const nsACString &key, + uint32_t typeBits); + + nsresult CacheOpportunistically(const nsCString &clientID, + const nsACString &key); + nsresult GetTypes(const nsCString &clientID, + const nsACString &key, + uint32_t *typeBits); + + nsresult GetMatchingNamespace(const nsCString &clientID, + const nsACString &key, + nsIApplicationCacheNamespace **out); + nsresult GatherEntries(const nsCString &clientID, + uint32_t typeBits, + uint32_t *count, + char *** values); + nsresult AddNamespace(const nsCString &clientID, + nsIApplicationCacheNamespace *ns); + + nsresult GetUsage(const nsACString &clientID, + uint32_t *usage); + + nsresult RunSimpleQuery(mozIStorageStatement *statment, + uint32_t resultIndex, + uint32_t * count, + char *** values); + + nsCOMPtr<mozIStorageConnection> mDB; + RefPtr<nsOfflineCacheEvictionFunction> mEvictionFunction; + + nsCOMPtr<mozIStorageStatement> mStatement_CacheSize; + nsCOMPtr<mozIStorageStatement> mStatement_ApplicationCacheSize; + nsCOMPtr<mozIStorageStatement> mStatement_EntryCount; + nsCOMPtr<mozIStorageStatement> mStatement_UpdateEntry; + nsCOMPtr<mozIStorageStatement> mStatement_UpdateEntrySize; + nsCOMPtr<mozIStorageStatement> mStatement_DeleteEntry; + nsCOMPtr<mozIStorageStatement> mStatement_FindEntry; + nsCOMPtr<mozIStorageStatement> mStatement_BindEntry; + nsCOMPtr<mozIStorageStatement> mStatement_ClearDomain; + nsCOMPtr<mozIStorageStatement> mStatement_MarkEntry; + nsCOMPtr<mozIStorageStatement> mStatement_UnmarkEntry; + nsCOMPtr<mozIStorageStatement> mStatement_GetTypes; + nsCOMPtr<mozIStorageStatement> mStatement_FindNamespaceEntry; + nsCOMPtr<mozIStorageStatement> mStatement_InsertNamespaceEntry; + nsCOMPtr<mozIStorageStatement> mStatement_CleanupUnmarked; + nsCOMPtr<mozIStorageStatement> mStatement_GatherEntries; + nsCOMPtr<mozIStorageStatement> mStatement_ActivateClient; + nsCOMPtr<mozIStorageStatement> mStatement_DeactivateGroup; + nsCOMPtr<mozIStorageStatement> mStatement_FindClient; + nsCOMPtr<mozIStorageStatement> mStatement_FindClientByNamespace; + nsCOMPtr<mozIStorageStatement> mStatement_EnumerateApps; + nsCOMPtr<mozIStorageStatement> mStatement_EnumerateGroups; + nsCOMPtr<mozIStorageStatement> mStatement_EnumerateGroupsTimeOrder; + + nsCOMPtr<nsIFile> mBaseDirectory; + nsCOMPtr<nsIFile> mCacheDirectory; + uint32_t mCacheCapacity; // in bytes + int32_t mDeltaCounter; + bool mAutoShutdown; + + mozilla::Mutex mLock; + + nsInterfaceHashtable<nsCStringHashKey, nsIWeakReference> mCaches; + nsClassHashtable<nsCStringHashKey, nsCString> mActiveCachesByGroup; + nsTHashtable<nsCStringHashKey> mActiveCaches; + nsTHashtable<nsCStringHashKey> mLockedEntries; + + nsCOMPtr<nsIThread> mInitThread; +}; + +#endif // nsOfflineCacheDevice_h__ diff --git a/netwerk/cache/nsDiskCacheEntry.cpp b/netwerk/cache/nsDiskCacheEntry.cpp new file mode 100644 index 000000000..cd5a34977 --- /dev/null +++ b/netwerk/cache/nsDiskCacheEntry.cpp @@ -0,0 +1,133 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsCache.h" +#include "nsDiskCache.h" +#include "nsDiskCacheEntry.h" +#include "nsDiskCacheBinding.h" +#include "nsCRT.h" + +#include "nsISerializable.h" +#include "nsSerializationHelper.h" + +/****************************************************************************** + * nsDiskCacheEntry + *****************************************************************************/ + +/** + * CreateCacheEntry() + * + * Creates an nsCacheEntry and sets all fields except for the binding. + */ +nsCacheEntry * +nsDiskCacheEntry::CreateCacheEntry(nsCacheDevice * device) +{ + nsCacheEntry * entry = nullptr; + nsresult rv = nsCacheEntry::Create(Key(), + nsICache::STREAM_BASED, + nsICache::STORE_ON_DISK, + device, + &entry); + if (NS_FAILED(rv) || !entry) return nullptr; + + entry->SetFetchCount(mFetchCount); + entry->SetLastFetched(mLastFetched); + entry->SetLastModified(mLastModified); + entry->SetExpirationTime(mExpirationTime); + entry->SetCacheDevice(device); + // XXX why does nsCacheService have to fill out device in BindEntry()? + entry->SetDataSize(mDataSize); + + rv = entry->UnflattenMetaData(MetaData(), mMetaDataSize); + if (NS_FAILED(rv)) { + delete entry; + return nullptr; + } + + // Restore security info, if present + const char* info = entry->GetMetaDataElement("security-info"); + if (info) { + nsCOMPtr<nsISupports> infoObj; + rv = NS_DeserializeObject(nsDependentCString(info), + getter_AddRefs(infoObj)); + if (NS_FAILED(rv)) { + delete entry; + return nullptr; + } + entry->SetSecurityInfo(infoObj); + } + + return entry; +} + + +/****************************************************************************** + * nsDiskCacheEntryInfo + *****************************************************************************/ + +NS_IMPL_ISUPPORTS(nsDiskCacheEntryInfo, nsICacheEntryInfo) + +NS_IMETHODIMP nsDiskCacheEntryInfo::GetClientID(char ** clientID) +{ + NS_ENSURE_ARG_POINTER(clientID); + return ClientIDFromCacheKey(nsDependentCString(mDiskEntry->Key()), clientID); +} + +extern const char DISK_CACHE_DEVICE_ID[]; +NS_IMETHODIMP nsDiskCacheEntryInfo::GetDeviceID(char ** deviceID) +{ + NS_ENSURE_ARG_POINTER(deviceID); + *deviceID = NS_strdup(mDeviceID); + return *deviceID ? NS_OK : NS_ERROR_OUT_OF_MEMORY; +} + + +NS_IMETHODIMP nsDiskCacheEntryInfo::GetKey(nsACString &clientKey) +{ + return ClientKeyFromCacheKey(nsDependentCString(mDiskEntry->Key()), clientKey); +} + +NS_IMETHODIMP nsDiskCacheEntryInfo::GetFetchCount(int32_t *aFetchCount) +{ + NS_ENSURE_ARG_POINTER(aFetchCount); + *aFetchCount = mDiskEntry->mFetchCount; + return NS_OK; +} + +NS_IMETHODIMP nsDiskCacheEntryInfo::GetLastFetched(uint32_t *aLastFetched) +{ + NS_ENSURE_ARG_POINTER(aLastFetched); + *aLastFetched = mDiskEntry->mLastFetched; + return NS_OK; +} + +NS_IMETHODIMP nsDiskCacheEntryInfo::GetLastModified(uint32_t *aLastModified) +{ + NS_ENSURE_ARG_POINTER(aLastModified); + *aLastModified = mDiskEntry->mLastModified; + return NS_OK; +} + +NS_IMETHODIMP nsDiskCacheEntryInfo::GetExpirationTime(uint32_t *aExpirationTime) +{ + NS_ENSURE_ARG_POINTER(aExpirationTime); + *aExpirationTime = mDiskEntry->mExpirationTime; + return NS_OK; +} + +NS_IMETHODIMP nsDiskCacheEntryInfo::IsStreamBased(bool *aStreamBased) +{ + NS_ENSURE_ARG_POINTER(aStreamBased); + *aStreamBased = true; + return NS_OK; +} + +NS_IMETHODIMP nsDiskCacheEntryInfo::GetDataSize(uint32_t *aDataSize) +{ + NS_ENSURE_ARG_POINTER(aDataSize); + *aDataSize = mDiskEntry->mDataSize; + return NS_OK; +} diff --git a/netwerk/cache/nsDiskCacheEntry.h b/netwerk/cache/nsDiskCacheEntry.h new file mode 100644 index 000000000..3bec33053 --- /dev/null +++ b/netwerk/cache/nsDiskCacheEntry.h @@ -0,0 +1,100 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef _nsDiskCacheEntry_h_ +#define _nsDiskCacheEntry_h_ + +#include "nsDiskCacheMap.h" + +#include "nsCacheEntry.h" + + +/****************************************************************************** + * nsDiskCacheEntry + *****************************************************************************/ +struct nsDiskCacheEntry { + uint32_t mHeaderVersion; // useful for stand-alone metadata files + uint32_t mMetaLocation; // for verification + int32_t mFetchCount; + uint32_t mLastFetched; + uint32_t mLastModified; + uint32_t mExpirationTime; + uint32_t mDataSize; + uint32_t mKeySize; // includes terminating null byte + uint32_t mMetaDataSize; // includes terminating null byte + // followed by key data (mKeySize bytes) + // followed by meta data (mMetaDataSize bytes) + + uint32_t Size() { return sizeof(nsDiskCacheEntry) + + mKeySize + mMetaDataSize; + } + + char* Key() { return reinterpret_cast<char*const>(this) + + sizeof(nsDiskCacheEntry); + } + + char* MetaData() + { return Key() + mKeySize; } + + nsCacheEntry * CreateCacheEntry(nsCacheDevice * device); + + void Swap() // host to network (memory to disk) + { +#if defined(IS_LITTLE_ENDIAN) + mHeaderVersion = htonl(mHeaderVersion); + mMetaLocation = htonl(mMetaLocation); + mFetchCount = htonl(mFetchCount); + mLastFetched = htonl(mLastFetched); + mLastModified = htonl(mLastModified); + mExpirationTime = htonl(mExpirationTime); + mDataSize = htonl(mDataSize); + mKeySize = htonl(mKeySize); + mMetaDataSize = htonl(mMetaDataSize); +#endif + } + + void Unswap() // network to host (disk to memory) + { +#if defined(IS_LITTLE_ENDIAN) + mHeaderVersion = ntohl(mHeaderVersion); + mMetaLocation = ntohl(mMetaLocation); + mFetchCount = ntohl(mFetchCount); + mLastFetched = ntohl(mLastFetched); + mLastModified = ntohl(mLastModified); + mExpirationTime = ntohl(mExpirationTime); + mDataSize = ntohl(mDataSize); + mKeySize = ntohl(mKeySize); + mMetaDataSize = ntohl(mMetaDataSize); +#endif + } +}; + + +/****************************************************************************** + * nsDiskCacheEntryInfo + *****************************************************************************/ +class nsDiskCacheEntryInfo : public nsICacheEntryInfo { +public: + NS_DECL_ISUPPORTS + NS_DECL_NSICACHEENTRYINFO + + nsDiskCacheEntryInfo(const char * deviceID, nsDiskCacheEntry * diskEntry) + : mDeviceID(deviceID) + , mDiskEntry(diskEntry) + { + } + + const char* Key() { return mDiskEntry->Key(); } + +private: + virtual ~nsDiskCacheEntryInfo() {} + + const char * mDeviceID; + nsDiskCacheEntry * mDiskEntry; +}; + + +#endif /* _nsDiskCacheEntry_h_ */ diff --git a/netwerk/cache/nsDiskCacheMap.cpp b/netwerk/cache/nsDiskCacheMap.cpp new file mode 100644 index 000000000..d7ce13a35 --- /dev/null +++ b/netwerk/cache/nsDiskCacheMap.cpp @@ -0,0 +1,1430 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* vim:set ts=4 sw=4 sts=4 cin et: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsCache.h" +#include "nsDiskCacheMap.h" +#include "nsDiskCacheBinding.h" +#include "nsDiskCacheEntry.h" +#include "nsDiskCacheDevice.h" +#include "nsCacheService.h" + +#include <string.h> +#include "nsPrintfCString.h" + +#include "nsISerializable.h" +#include "nsSerializationHelper.h" + +#include "mozilla/MemoryReporting.h" +#include "mozilla/Sprintf.h" +#include "mozilla/Telemetry.h" +#include <algorithm> + +using namespace mozilla; + +/****************************************************************************** + * nsDiskCacheMap + *****************************************************************************/ + +/** + * File operations + */ + +nsresult +nsDiskCacheMap::Open(nsIFile * cacheDirectory, + nsDiskCache::CorruptCacheInfo * corruptInfo) +{ + NS_ENSURE_ARG_POINTER(corruptInfo); + + // Assume we have an unexpected error until we find otherwise. + *corruptInfo = nsDiskCache::kUnexpectedError; + NS_ENSURE_ARG_POINTER(cacheDirectory); + if (mMapFD) return NS_ERROR_ALREADY_INITIALIZED; + + mCacheDirectory = cacheDirectory; // save a reference for ourselves + + // create nsIFile for _CACHE_MAP_ + nsresult rv; + nsCOMPtr<nsIFile> file; + rv = cacheDirectory->Clone(getter_AddRefs(file)); + rv = file->AppendNative(NS_LITERAL_CSTRING("_CACHE_MAP_")); + NS_ENSURE_SUCCESS(rv, rv); + + // open the file - restricted to user, the data could be confidential + rv = file->OpenNSPRFileDesc(PR_RDWR | PR_CREATE_FILE, 00600, &mMapFD); + if (NS_FAILED(rv)) { + *corruptInfo = nsDiskCache::kOpenCacheMapError; + NS_WARNING("Could not open cache map file"); + return NS_ERROR_FILE_CORRUPTED; + } + + bool cacheFilesExist = CacheFilesExist(); + rv = NS_ERROR_FILE_CORRUPTED; // presume the worst + uint32_t mapSize = PR_Available(mMapFD); + + if (NS_FAILED(InitCacheClean(cacheDirectory, + corruptInfo))) { + // corruptInfo is set in the call to InitCacheClean + goto error_exit; + } + + // check size of map file + if (mapSize == 0) { // creating a new _CACHE_MAP_ + + // block files shouldn't exist if we're creating the _CACHE_MAP_ + if (cacheFilesExist) { + *corruptInfo = nsDiskCache::kBlockFilesShouldNotExist; + goto error_exit; + } + + if (NS_FAILED(CreateCacheSubDirectories())) { + *corruptInfo = nsDiskCache::kCreateCacheSubdirectories; + goto error_exit; + } + + // create the file - initialize in memory + memset(&mHeader, 0, sizeof(nsDiskCacheHeader)); + mHeader.mVersion = nsDiskCache::kCurrentVersion; + mHeader.mRecordCount = kMinRecordCount; + mRecordArray = (nsDiskCacheRecord *) + PR_CALLOC(mHeader.mRecordCount * sizeof(nsDiskCacheRecord)); + if (!mRecordArray) { + *corruptInfo = nsDiskCache::kOutOfMemory; + rv = NS_ERROR_OUT_OF_MEMORY; + goto error_exit; + } + } else if (mapSize >= sizeof(nsDiskCacheHeader)) { // read existing _CACHE_MAP_ + + // if _CACHE_MAP_ exists, so should the block files + if (!cacheFilesExist) { + *corruptInfo = nsDiskCache::kBlockFilesShouldExist; + goto error_exit; + } + + CACHE_LOG_DEBUG(("CACHE: nsDiskCacheMap::Open [this=%p] reading map", this)); + + // read the header + uint32_t bytesRead = PR_Read(mMapFD, &mHeader, sizeof(nsDiskCacheHeader)); + if (sizeof(nsDiskCacheHeader) != bytesRead) { + *corruptInfo = nsDiskCache::kHeaderSizeNotRead; + goto error_exit; + } + mHeader.Unswap(); + + if (mHeader.mIsDirty) { + *corruptInfo = nsDiskCache::kHeaderIsDirty; + goto error_exit; + } + + if (mHeader.mVersion != nsDiskCache::kCurrentVersion) { + *corruptInfo = nsDiskCache::kVersionMismatch; + goto error_exit; + } + + uint32_t recordArraySize = + mHeader.mRecordCount * sizeof(nsDiskCacheRecord); + if (mapSize < recordArraySize + sizeof(nsDiskCacheHeader)) { + *corruptInfo = nsDiskCache::kRecordsIncomplete; + goto error_exit; + } + + // Get the space for the records + mRecordArray = (nsDiskCacheRecord *) PR_MALLOC(recordArraySize); + if (!mRecordArray) { + *corruptInfo = nsDiskCache::kOutOfMemory; + rv = NS_ERROR_OUT_OF_MEMORY; + goto error_exit; + } + + // Read the records + bytesRead = PR_Read(mMapFD, mRecordArray, recordArraySize); + if (bytesRead < recordArraySize) { + *corruptInfo = nsDiskCache::kNotEnoughToRead; + goto error_exit; + } + + // Unswap each record + int32_t total = 0; + for (int32_t i = 0; i < mHeader.mRecordCount; ++i) { + if (mRecordArray[i].HashNumber()) { +#if defined(IS_LITTLE_ENDIAN) + mRecordArray[i].Unswap(); +#endif + total ++; + } + } + + // verify entry count + if (total != mHeader.mEntryCount) { + *corruptInfo = nsDiskCache::kEntryCountIncorrect; + goto error_exit; + } + + } else { + *corruptInfo = nsDiskCache::kHeaderIncomplete; + goto error_exit; + } + + rv = OpenBlockFiles(corruptInfo); + if (NS_FAILED(rv)) { + // corruptInfo is set in the call to OpenBlockFiles + goto error_exit; + } + + // set dirty bit and flush header + mHeader.mIsDirty = true; + rv = FlushHeader(); + if (NS_FAILED(rv)) { + *corruptInfo = nsDiskCache::kFlushHeaderError; + goto error_exit; + } + + Telemetry::Accumulate(Telemetry::HTTP_DISK_CACHE_OVERHEAD, + (uint32_t)SizeOfExcludingThis(moz_malloc_size_of)); + + *corruptInfo = nsDiskCache::kNotCorrupt; + return NS_OK; + +error_exit: + (void) Close(false); + + return rv; +} + + +nsresult +nsDiskCacheMap::Close(bool flush) +{ + nsCacheService::AssertOwnsLock(); + nsresult rv = NS_OK; + + // Cancel any pending cache validation event, the FlushRecords call below + // will validate the cache. + if (mCleanCacheTimer) { + mCleanCacheTimer->Cancel(); + } + + // If cache map file and its block files are still open, close them + if (mMapFD) { + // close block files + rv = CloseBlockFiles(flush); + if (NS_SUCCEEDED(rv) && flush && mRecordArray) { + // write the map records + rv = FlushRecords(false); // don't bother swapping buckets back + if (NS_SUCCEEDED(rv)) { + // clear dirty bit + mHeader.mIsDirty = false; + rv = FlushHeader(); + } + } + if ((PR_Close(mMapFD) != PR_SUCCESS) && (NS_SUCCEEDED(rv))) + rv = NS_ERROR_UNEXPECTED; + + mMapFD = nullptr; + } + + if (mCleanFD) { + PR_Close(mCleanFD); + mCleanFD = nullptr; + } + + PR_FREEIF(mRecordArray); + PR_FREEIF(mBuffer); + mBufferSize = 0; + return rv; +} + + +nsresult +nsDiskCacheMap::Trim() +{ + nsresult rv, rv2 = NS_OK; + for (int i=0; i < kNumBlockFiles; ++i) { + rv = mBlockFile[i].Trim(); + if (NS_FAILED(rv)) rv2 = rv; // if one or more errors, report at least one + } + // Try to shrink the records array + rv = ShrinkRecords(); + if (NS_FAILED(rv)) rv2 = rv; // if one or more errors, report at least one + return rv2; +} + + +nsresult +nsDiskCacheMap::FlushHeader() +{ + if (!mMapFD) return NS_ERROR_NOT_AVAILABLE; + + // seek to beginning of cache map + int32_t filePos = PR_Seek(mMapFD, 0, PR_SEEK_SET); + if (filePos != 0) return NS_ERROR_UNEXPECTED; + + // write the header + mHeader.Swap(); + int32_t bytesWritten = PR_Write(mMapFD, &mHeader, sizeof(nsDiskCacheHeader)); + mHeader.Unswap(); + if (sizeof(nsDiskCacheHeader) != bytesWritten) { + return NS_ERROR_UNEXPECTED; + } + + PRStatus err = PR_Sync(mMapFD); + if (err != PR_SUCCESS) return NS_ERROR_UNEXPECTED; + + // If we have a clean header then revalidate the cache clean file + if (!mHeader.mIsDirty) { + RevalidateCache(); + } + + return NS_OK; +} + + +nsresult +nsDiskCacheMap::FlushRecords(bool unswap) +{ + if (!mMapFD) return NS_ERROR_NOT_AVAILABLE; + + // seek to beginning of buckets + int32_t filePos = PR_Seek(mMapFD, sizeof(nsDiskCacheHeader), PR_SEEK_SET); + if (filePos != sizeof(nsDiskCacheHeader)) + return NS_ERROR_UNEXPECTED; + +#if defined(IS_LITTLE_ENDIAN) + // Swap each record + for (int32_t i = 0; i < mHeader.mRecordCount; ++i) { + if (mRecordArray[i].HashNumber()) + mRecordArray[i].Swap(); + } +#endif + + int32_t recordArraySize = sizeof(nsDiskCacheRecord) * mHeader.mRecordCount; + + int32_t bytesWritten = PR_Write(mMapFD, mRecordArray, recordArraySize); + if (bytesWritten != recordArraySize) + return NS_ERROR_UNEXPECTED; + +#if defined(IS_LITTLE_ENDIAN) + if (unswap) { + // Unswap each record + for (int32_t i = 0; i < mHeader.mRecordCount; ++i) { + if (mRecordArray[i].HashNumber()) + mRecordArray[i].Unswap(); + } + } +#endif + + return NS_OK; +} + + +/** + * Record operations + */ + +uint32_t +nsDiskCacheMap::GetBucketRank(uint32_t bucketIndex, uint32_t targetRank) +{ + nsDiskCacheRecord * records = GetFirstRecordInBucket(bucketIndex); + uint32_t rank = 0; + + for (int i = mHeader.mBucketUsage[bucketIndex]-1; i >= 0; i--) { + if ((rank < records[i].EvictionRank()) && + ((targetRank == 0) || (records[i].EvictionRank() < targetRank))) + rank = records[i].EvictionRank(); + } + return rank; +} + +nsresult +nsDiskCacheMap::GrowRecords() +{ + if (mHeader.mRecordCount >= mMaxRecordCount) + return NS_OK; + CACHE_LOG_DEBUG(("CACHE: GrowRecords\n")); + + // Resize the record array + int32_t newCount = mHeader.mRecordCount << 1; + if (newCount > mMaxRecordCount) + newCount = mMaxRecordCount; + nsDiskCacheRecord *newArray = (nsDiskCacheRecord *) + PR_REALLOC(mRecordArray, newCount * sizeof(nsDiskCacheRecord)); + if (!newArray) + return NS_ERROR_OUT_OF_MEMORY; + + // Space out the buckets + uint32_t oldRecordsPerBucket = GetRecordsPerBucket(); + uint32_t newRecordsPerBucket = newCount / kBuckets; + // Work from back to space out each bucket to the new array + for (int bucketIndex = kBuckets - 1; bucketIndex >= 0; --bucketIndex) { + // Move bucket + nsDiskCacheRecord *newRecords = newArray + bucketIndex * newRecordsPerBucket; + const uint32_t count = mHeader.mBucketUsage[bucketIndex]; + memmove(newRecords, + newArray + bucketIndex * oldRecordsPerBucket, + count * sizeof(nsDiskCacheRecord)); + // clear unused records + memset(newRecords + count, 0, + (newRecordsPerBucket - count) * sizeof(nsDiskCacheRecord)); + } + + // Set as the new record array + mRecordArray = newArray; + mHeader.mRecordCount = newCount; + + InvalidateCache(); + + return NS_OK; +} + +nsresult +nsDiskCacheMap::ShrinkRecords() +{ + if (mHeader.mRecordCount <= kMinRecordCount) + return NS_OK; + CACHE_LOG_DEBUG(("CACHE: ShrinkRecords\n")); + + // Verify if we can shrink the record array: all buckets must be less than + // 1/2 filled + uint32_t maxUsage = 0, bucketIndex; + for (bucketIndex = 0; bucketIndex < kBuckets; ++bucketIndex) { + if (maxUsage < mHeader.mBucketUsage[bucketIndex]) + maxUsage = mHeader.mBucketUsage[bucketIndex]; + } + // Determine new bucket size, halve size until maxUsage + uint32_t oldRecordsPerBucket = GetRecordsPerBucket(); + uint32_t newRecordsPerBucket = oldRecordsPerBucket; + while (maxUsage < (newRecordsPerBucket >> 1)) + newRecordsPerBucket >>= 1; + if (newRecordsPerBucket < (kMinRecordCount / kBuckets)) + newRecordsPerBucket = (kMinRecordCount / kBuckets); + NS_ASSERTION(newRecordsPerBucket <= oldRecordsPerBucket, + "ShrinkRecords() can't grow records!"); + if (newRecordsPerBucket == oldRecordsPerBucket) + return NS_OK; + // Move the buckets close to each other + for (bucketIndex = 1; bucketIndex < kBuckets; ++bucketIndex) { + // Move bucket + memmove(mRecordArray + bucketIndex * newRecordsPerBucket, + mRecordArray + bucketIndex * oldRecordsPerBucket, + newRecordsPerBucket * sizeof(nsDiskCacheRecord)); + } + + // Shrink the record array memory block itself + uint32_t newCount = newRecordsPerBucket * kBuckets; + nsDiskCacheRecord* newArray = (nsDiskCacheRecord *) + PR_REALLOC(mRecordArray, newCount * sizeof(nsDiskCacheRecord)); + if (!newArray) + return NS_ERROR_OUT_OF_MEMORY; + + // Set as the new record array + mRecordArray = newArray; + mHeader.mRecordCount = newCount; + + InvalidateCache(); + + return NS_OK; +} + +nsresult +nsDiskCacheMap::AddRecord( nsDiskCacheRecord * mapRecord, + nsDiskCacheRecord * oldRecord) +{ + CACHE_LOG_DEBUG(("CACHE: AddRecord [%x]\n", mapRecord->HashNumber())); + + const uint32_t hashNumber = mapRecord->HashNumber(); + const uint32_t bucketIndex = GetBucketIndex(hashNumber); + const uint32_t count = mHeader.mBucketUsage[bucketIndex]; + + oldRecord->SetHashNumber(0); // signify no record + + if (count == GetRecordsPerBucket()) { + // Ignore failure to grow the record space, we will then reuse old records + GrowRecords(); + } + + nsDiskCacheRecord * records = GetFirstRecordInBucket(bucketIndex); + if (count < GetRecordsPerBucket()) { + // stick the new record at the end + records[count] = *mapRecord; + mHeader.mEntryCount++; + mHeader.mBucketUsage[bucketIndex]++; + if (mHeader.mEvictionRank[bucketIndex] < mapRecord->EvictionRank()) + mHeader.mEvictionRank[bucketIndex] = mapRecord->EvictionRank(); + InvalidateCache(); + } else { + // Find the record with the highest eviction rank + nsDiskCacheRecord * mostEvictable = &records[0]; + for (int i = count-1; i > 0; i--) { + if (records[i].EvictionRank() > mostEvictable->EvictionRank()) + mostEvictable = &records[i]; + } + *oldRecord = *mostEvictable; // i == GetRecordsPerBucket(), so + // evict the mostEvictable + *mostEvictable = *mapRecord; // replace it with the new record + // check if we need to update mostEvictable entry in header + if (mHeader.mEvictionRank[bucketIndex] < mapRecord->EvictionRank()) + mHeader.mEvictionRank[bucketIndex] = mapRecord->EvictionRank(); + if (oldRecord->EvictionRank() >= mHeader.mEvictionRank[bucketIndex]) + mHeader.mEvictionRank[bucketIndex] = GetBucketRank(bucketIndex, 0); + InvalidateCache(); + } + + NS_ASSERTION(mHeader.mEvictionRank[bucketIndex] == GetBucketRank(bucketIndex, 0), + "eviction rank out of sync"); + return NS_OK; +} + + +nsresult +nsDiskCacheMap::UpdateRecord( nsDiskCacheRecord * mapRecord) +{ + CACHE_LOG_DEBUG(("CACHE: UpdateRecord [%x]\n", mapRecord->HashNumber())); + + const uint32_t hashNumber = mapRecord->HashNumber(); + const uint32_t bucketIndex = GetBucketIndex(hashNumber); + nsDiskCacheRecord * records = GetFirstRecordInBucket(bucketIndex); + + for (int i = mHeader.mBucketUsage[bucketIndex]-1; i >= 0; i--) { + if (records[i].HashNumber() == hashNumber) { + const uint32_t oldRank = records[i].EvictionRank(); + + // stick the new record here + records[i] = *mapRecord; + + // update eviction rank in header if necessary + if (mHeader.mEvictionRank[bucketIndex] < mapRecord->EvictionRank()) + mHeader.mEvictionRank[bucketIndex] = mapRecord->EvictionRank(); + else if (mHeader.mEvictionRank[bucketIndex] == oldRank) + mHeader.mEvictionRank[bucketIndex] = GetBucketRank(bucketIndex, 0); + + InvalidateCache(); + +NS_ASSERTION(mHeader.mEvictionRank[bucketIndex] == GetBucketRank(bucketIndex, 0), + "eviction rank out of sync"); + return NS_OK; + } + } + NS_NOTREACHED("record not found"); + return NS_ERROR_UNEXPECTED; +} + + +nsresult +nsDiskCacheMap::FindRecord( uint32_t hashNumber, nsDiskCacheRecord * result) +{ + const uint32_t bucketIndex = GetBucketIndex(hashNumber); + nsDiskCacheRecord * records = GetFirstRecordInBucket(bucketIndex); + + for (int i = mHeader.mBucketUsage[bucketIndex]-1; i >= 0; i--) { + if (records[i].HashNumber() == hashNumber) { + *result = records[i]; // copy the record + NS_ASSERTION(result->ValidRecord(), "bad cache map record"); + return NS_OK; + } + } + return NS_ERROR_CACHE_KEY_NOT_FOUND; +} + + +nsresult +nsDiskCacheMap::DeleteRecord( nsDiskCacheRecord * mapRecord) +{ + CACHE_LOG_DEBUG(("CACHE: DeleteRecord [%x]\n", mapRecord->HashNumber())); + + const uint32_t hashNumber = mapRecord->HashNumber(); + const uint32_t bucketIndex = GetBucketIndex(hashNumber); + nsDiskCacheRecord * records = GetFirstRecordInBucket(bucketIndex); + uint32_t last = mHeader.mBucketUsage[bucketIndex]-1; + + for (int i = last; i >= 0; i--) { + if (records[i].HashNumber() == hashNumber) { + // found it, now delete it. + uint32_t evictionRank = records[i].EvictionRank(); + NS_ASSERTION(evictionRank == mapRecord->EvictionRank(), + "evictionRank out of sync"); + // if not the last record, shift last record into opening + records[i] = records[last]; + records[last].SetHashNumber(0); // clear last record + mHeader.mBucketUsage[bucketIndex] = last; + mHeader.mEntryCount--; + + // update eviction rank + uint32_t bucketIndex = GetBucketIndex(mapRecord->HashNumber()); + if (mHeader.mEvictionRank[bucketIndex] <= evictionRank) { + mHeader.mEvictionRank[bucketIndex] = GetBucketRank(bucketIndex, 0); + } + + InvalidateCache(); + + NS_ASSERTION(mHeader.mEvictionRank[bucketIndex] == + GetBucketRank(bucketIndex, 0), "eviction rank out of sync"); + return NS_OK; + } + } + return NS_ERROR_UNEXPECTED; +} + + +int32_t +nsDiskCacheMap::VisitEachRecord(uint32_t bucketIndex, + nsDiskCacheRecordVisitor * visitor, + uint32_t evictionRank) +{ + int32_t rv = kVisitNextRecord; + uint32_t count = mHeader.mBucketUsage[bucketIndex]; + nsDiskCacheRecord * records = GetFirstRecordInBucket(bucketIndex); + + // call visitor for each entry (matching any eviction rank) + for (int i = count-1; i >= 0; i--) { + if (evictionRank > records[i].EvictionRank()) continue; + + rv = visitor->VisitRecord(&records[i]); + if (rv == kStopVisitingRecords) + break; // Stop visiting records + + if (rv == kDeleteRecordAndContinue) { + --count; + records[i] = records[count]; + records[count].SetHashNumber(0); + InvalidateCache(); + } + } + + if (mHeader.mBucketUsage[bucketIndex] - count != 0) { + mHeader.mEntryCount -= mHeader.mBucketUsage[bucketIndex] - count; + mHeader.mBucketUsage[bucketIndex] = count; + // recalc eviction rank + mHeader.mEvictionRank[bucketIndex] = GetBucketRank(bucketIndex, 0); + } + NS_ASSERTION(mHeader.mEvictionRank[bucketIndex] == + GetBucketRank(bucketIndex, 0), "eviction rank out of sync"); + + return rv; +} + + +/** + * VisitRecords + * + * Visit every record in cache map in the most convenient order + */ +nsresult +nsDiskCacheMap::VisitRecords( nsDiskCacheRecordVisitor * visitor) +{ + for (int bucketIndex = 0; bucketIndex < kBuckets; ++bucketIndex) { + if (VisitEachRecord(bucketIndex, visitor, 0) == kStopVisitingRecords) + break; + } + return NS_OK; +} + + +/** + * EvictRecords + * + * Just like VisitRecords, but visits the records in order of their eviction rank + */ +nsresult +nsDiskCacheMap::EvictRecords( nsDiskCacheRecordVisitor * visitor) +{ + uint32_t tempRank[kBuckets]; + int bucketIndex = 0; + + // copy eviction rank array + for (bucketIndex = 0; bucketIndex < kBuckets; ++bucketIndex) + tempRank[bucketIndex] = mHeader.mEvictionRank[bucketIndex]; + + // Maximum number of iterations determined by number of records + // as a safety limiter for the loop. Use a copy of mHeader.mEntryCount since + // the value could decrease if some entry is evicted. + int32_t entryCount = mHeader.mEntryCount; + for (int n = 0; n < entryCount; ++n) { + + // find bucket with highest eviction rank + uint32_t rank = 0; + for (int i = 0; i < kBuckets; ++i) { + if (rank < tempRank[i]) { + rank = tempRank[i]; + bucketIndex = i; + } + } + + if (rank == 0) break; // we've examined all the records + + // visit records in bucket with eviction ranks >= target eviction rank + if (VisitEachRecord(bucketIndex, visitor, rank) == kStopVisitingRecords) + break; + + // find greatest rank less than 'rank' + tempRank[bucketIndex] = GetBucketRank(bucketIndex, rank); + } + return NS_OK; +} + + + +nsresult +nsDiskCacheMap::OpenBlockFiles(nsDiskCache::CorruptCacheInfo * corruptInfo) +{ + NS_ENSURE_ARG_POINTER(corruptInfo); + + // create nsIFile for block file + nsCOMPtr<nsIFile> blockFile; + nsresult rv = NS_OK; + *corruptInfo = nsDiskCache::kUnexpectedError; + + for (int i = 0; i < kNumBlockFiles; ++i) { + rv = GetBlockFileForIndex(i, getter_AddRefs(blockFile)); + if (NS_FAILED(rv)) { + *corruptInfo = nsDiskCache::kCouldNotGetBlockFileForIndex; + break; + } + + uint32_t blockSize = GetBlockSizeForIndex(i+1); // +1 to match file selectors 1,2,3 + uint32_t bitMapSize = GetBitMapSizeForIndex(i+1); + rv = mBlockFile[i].Open(blockFile, blockSize, bitMapSize, corruptInfo); + if (NS_FAILED(rv)) { + // corruptInfo was set inside the call to mBlockFile[i].Open + break; + } + } + // close all files in case of any error + if (NS_FAILED(rv)) + (void)CloseBlockFiles(false); // we already have an error to report + + return rv; +} + + +nsresult +nsDiskCacheMap::CloseBlockFiles(bool flush) +{ + nsresult rv, rv2 = NS_OK; + for (int i=0; i < kNumBlockFiles; ++i) { + rv = mBlockFile[i].Close(flush); + if (NS_FAILED(rv)) rv2 = rv; // if one or more errors, report at least one + } + return rv2; +} + + +bool +nsDiskCacheMap::CacheFilesExist() +{ + nsCOMPtr<nsIFile> blockFile; + nsresult rv; + + for (int i = 0; i < kNumBlockFiles; ++i) { + bool exists; + rv = GetBlockFileForIndex(i, getter_AddRefs(blockFile)); + if (NS_FAILED(rv)) return false; + + rv = blockFile->Exists(&exists); + if (NS_FAILED(rv) || !exists) return false; + } + + return true; +} + + +nsresult +nsDiskCacheMap::CreateCacheSubDirectories() +{ + if (!mCacheDirectory) + return NS_ERROR_UNEXPECTED; + + for (int32_t index = 0 ; index < 16 ; index++) { + nsCOMPtr<nsIFile> file; + nsresult rv = mCacheDirectory->Clone(getter_AddRefs(file)); + if (NS_FAILED(rv)) + return rv; + + rv = file->AppendNative(nsPrintfCString("%X", index)); + if (NS_FAILED(rv)) + return rv; + + rv = file->Create(nsIFile::DIRECTORY_TYPE, 0700); + if (NS_FAILED(rv)) + return rv; + } + + return NS_OK; +} + + +nsDiskCacheEntry * +nsDiskCacheMap::ReadDiskCacheEntry(nsDiskCacheRecord * record) +{ + CACHE_LOG_DEBUG(("CACHE: ReadDiskCacheEntry [%x]\n", record->HashNumber())); + + nsresult rv = NS_ERROR_UNEXPECTED; + nsDiskCacheEntry * diskEntry = nullptr; + uint32_t metaFile = record->MetaFile(); + int32_t bytesRead = 0; + + if (!record->MetaLocationInitialized()) return nullptr; + + if (metaFile == 0) { // entry/metadata stored in separate file + // open and read the file + nsCOMPtr<nsIFile> file; + rv = GetLocalFileForDiskCacheRecord(record, + nsDiskCache::kMetaData, + false, + getter_AddRefs(file)); + NS_ENSURE_SUCCESS(rv, nullptr); + + CACHE_LOG_DEBUG(("CACHE: nsDiskCacheMap::ReadDiskCacheEntry" + "[this=%p] reading disk cache entry", this)); + + PRFileDesc * fd = nullptr; + + // open the file - restricted to user, the data could be confidential + rv = file->OpenNSPRFileDesc(PR_RDONLY, 00600, &fd); + NS_ENSURE_SUCCESS(rv, nullptr); + + int32_t fileSize = PR_Available(fd); + if (fileSize < 0) { + // an error occurred. We could call PR_GetError(), but how would that help? + rv = NS_ERROR_UNEXPECTED; + } else { + rv = EnsureBuffer(fileSize); + if (NS_SUCCEEDED(rv)) { + bytesRead = PR_Read(fd, mBuffer, fileSize); + if (bytesRead < fileSize) { + rv = NS_ERROR_UNEXPECTED; + } + } + } + PR_Close(fd); + NS_ENSURE_SUCCESS(rv, nullptr); + + } else if (metaFile < (kNumBlockFiles + 1)) { + // entry/metadata stored in cache block file + + // allocate buffer + uint32_t blockCount = record->MetaBlockCount(); + bytesRead = blockCount * GetBlockSizeForIndex(metaFile); + + rv = EnsureBuffer(bytesRead); + NS_ENSURE_SUCCESS(rv, nullptr); + + // read diskEntry, note when the blocks are at the end of file, + // bytesRead may be less than blockSize*blockCount. + // But the bytesRead should at least agree with the real disk entry size. + rv = mBlockFile[metaFile - 1].ReadBlocks(mBuffer, + record->MetaStartBlock(), + blockCount, + &bytesRead); + NS_ENSURE_SUCCESS(rv, nullptr); + } + diskEntry = (nsDiskCacheEntry *)mBuffer; + diskEntry->Unswap(); // disk to memory + // Check if calculated size agrees with bytesRead + if (bytesRead < 0 || (uint32_t)bytesRead < diskEntry->Size()) + return nullptr; + + // Return the buffer containing the diskEntry structure + return diskEntry; +} + + +/** + * CreateDiskCacheEntry(nsCacheEntry * entry) + * + * Prepare an nsCacheEntry for writing to disk + */ +nsDiskCacheEntry * +nsDiskCacheMap::CreateDiskCacheEntry(nsDiskCacheBinding * binding, + uint32_t * aSize) +{ + nsCacheEntry * entry = binding->mCacheEntry; + if (!entry) return nullptr; + + // Store security info, if it is serializable + nsCOMPtr<nsISupports> infoObj = entry->SecurityInfo(); + nsCOMPtr<nsISerializable> serializable = do_QueryInterface(infoObj); + if (infoObj && !serializable) return nullptr; + if (serializable) { + nsCString info; + nsresult rv = NS_SerializeToString(serializable, info); + if (NS_FAILED(rv)) return nullptr; + rv = entry->SetMetaDataElement("security-info", info.get()); + if (NS_FAILED(rv)) return nullptr; + } + + uint32_t keySize = entry->Key()->Length() + 1; + uint32_t metaSize = entry->MetaDataSize(); + uint32_t size = sizeof(nsDiskCacheEntry) + keySize + metaSize; + + if (aSize) *aSize = size; + + nsresult rv = EnsureBuffer(size); + if (NS_FAILED(rv)) return nullptr; + + nsDiskCacheEntry *diskEntry = (nsDiskCacheEntry *)mBuffer; + diskEntry->mHeaderVersion = nsDiskCache::kCurrentVersion; + diskEntry->mMetaLocation = binding->mRecord.MetaLocation(); + diskEntry->mFetchCount = entry->FetchCount(); + diskEntry->mLastFetched = entry->LastFetched(); + diskEntry->mLastModified = entry->LastModified(); + diskEntry->mExpirationTime = entry->ExpirationTime(); + diskEntry->mDataSize = entry->DataSize(); + diskEntry->mKeySize = keySize; + diskEntry->mMetaDataSize = metaSize; + + memcpy(diskEntry->Key(), entry->Key()->get(), keySize); + + rv = entry->FlattenMetaData(diskEntry->MetaData(), metaSize); + if (NS_FAILED(rv)) return nullptr; + + return diskEntry; +} + + +nsresult +nsDiskCacheMap::WriteDiskCacheEntry(nsDiskCacheBinding * binding) +{ + CACHE_LOG_DEBUG(("CACHE: WriteDiskCacheEntry [%x]\n", + binding->mRecord.HashNumber())); + + nsresult rv = NS_OK; + uint32_t size; + nsDiskCacheEntry * diskEntry = CreateDiskCacheEntry(binding, &size); + if (!diskEntry) return NS_ERROR_UNEXPECTED; + + uint32_t fileIndex = CalculateFileIndex(size); + + // Deallocate old storage if necessary + if (binding->mRecord.MetaLocationInitialized()) { + // we have existing storage + + if ((binding->mRecord.MetaFile() == 0) && + (fileIndex == 0)) { // keeping the separate file + // just decrement total + DecrementTotalSize(binding->mRecord.MetaFileSize()); + NS_ASSERTION(binding->mRecord.MetaFileGeneration() == binding->mGeneration, + "generations out of sync"); + } else { + rv = DeleteStorage(&binding->mRecord, nsDiskCache::kMetaData); + NS_ENSURE_SUCCESS(rv, rv); + } + } + + binding->mRecord.SetEvictionRank(ULONG_MAX - SecondsFromPRTime(PR_Now())); + // write entry data to disk cache block file + diskEntry->Swap(); + + if (fileIndex != 0) { + while (1) { + uint32_t blockSize = GetBlockSizeForIndex(fileIndex); + uint32_t blocks = ((size - 1) / blockSize) + 1; + + int32_t startBlock; + rv = mBlockFile[fileIndex - 1].WriteBlocks(diskEntry, size, blocks, + &startBlock); + if (NS_SUCCEEDED(rv)) { + // update binding and cache map record + binding->mRecord.SetMetaBlocks(fileIndex, startBlock, blocks); + + rv = UpdateRecord(&binding->mRecord); + NS_ENSURE_SUCCESS(rv, rv); + + // XXX we should probably write out bucket ourselves + + IncrementTotalSize(blocks, blockSize); + break; + } + + if (fileIndex == kNumBlockFiles) { + fileIndex = 0; // write data to separate file + break; + } + + // try next block file + fileIndex++; + } + } + + if (fileIndex == 0) { + // Write entry data to separate file + uint32_t metaFileSizeK = ((size + 0x03FF) >> 10); // round up to nearest 1k + if (metaFileSizeK > kMaxDataSizeK) + metaFileSizeK = kMaxDataSizeK; + + binding->mRecord.SetMetaFileGeneration(binding->mGeneration); + binding->mRecord.SetMetaFileSize(metaFileSizeK); + rv = UpdateRecord(&binding->mRecord); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr<nsIFile> localFile; + rv = GetLocalFileForDiskCacheRecord(&binding->mRecord, + nsDiskCache::kMetaData, + true, + getter_AddRefs(localFile)); + NS_ENSURE_SUCCESS(rv, rv); + + // open the file + PRFileDesc * fd; + // open the file - restricted to user, the data could be confidential + rv = localFile->OpenNSPRFileDesc(PR_RDWR | PR_TRUNCATE | PR_CREATE_FILE, 00600, &fd); + NS_ENSURE_SUCCESS(rv, rv); + + // write the file + int32_t bytesWritten = PR_Write(fd, diskEntry, size); + + PRStatus err = PR_Close(fd); + if ((bytesWritten != (int32_t)size) || (err != PR_SUCCESS)) { + return NS_ERROR_UNEXPECTED; + } + + IncrementTotalSize(metaFileSizeK); + } + + return rv; +} + + +nsresult +nsDiskCacheMap::ReadDataCacheBlocks(nsDiskCacheBinding * binding, char * buffer, uint32_t size) +{ + CACHE_LOG_DEBUG(("CACHE: ReadDataCacheBlocks [%x size=%u]\n", + binding->mRecord.HashNumber(), size)); + + uint32_t fileIndex = binding->mRecord.DataFile(); + int32_t readSize = size; + + nsresult rv = mBlockFile[fileIndex - 1].ReadBlocks(buffer, + binding->mRecord.DataStartBlock(), + binding->mRecord.DataBlockCount(), + &readSize); + NS_ENSURE_SUCCESS(rv, rv); + if (readSize < (int32_t)size) { + rv = NS_ERROR_UNEXPECTED; + } + return rv; +} + + +nsresult +nsDiskCacheMap::WriteDataCacheBlocks(nsDiskCacheBinding * binding, char * buffer, uint32_t size) +{ + CACHE_LOG_DEBUG(("CACHE: WriteDataCacheBlocks [%x size=%u]\n", + binding->mRecord.HashNumber(), size)); + + nsresult rv = NS_OK; + + // determine block file & number of blocks + uint32_t fileIndex = CalculateFileIndex(size); + uint32_t blockCount = 0; + int32_t startBlock = 0; + + if (size > 0) { + // if fileIndex is 0, bad things happen below, which makes gcc 4.7 + // complain, but it's not supposed to happen. See bug 854105. + MOZ_ASSERT(fileIndex); + while (fileIndex) { + uint32_t blockSize = GetBlockSizeForIndex(fileIndex); + blockCount = ((size - 1) / blockSize) + 1; + + rv = mBlockFile[fileIndex - 1].WriteBlocks(buffer, size, blockCount, + &startBlock); + if (NS_SUCCEEDED(rv)) { + IncrementTotalSize(blockCount, blockSize); + break; + } + + if (fileIndex == kNumBlockFiles) + return rv; + + fileIndex++; + } + } + + // update binding and cache map record + binding->mRecord.SetDataBlocks(fileIndex, startBlock, blockCount); + if (!binding->mDoomed) { + rv = UpdateRecord(&binding->mRecord); + } + return rv; +} + + +nsresult +nsDiskCacheMap::DeleteStorage(nsDiskCacheRecord * record) +{ + nsresult rv1 = DeleteStorage(record, nsDiskCache::kData); + nsresult rv2 = DeleteStorage(record, nsDiskCache::kMetaData); + return NS_FAILED(rv1) ? rv1 : rv2; +} + + +nsresult +nsDiskCacheMap::DeleteStorage(nsDiskCacheRecord * record, bool metaData) +{ + CACHE_LOG_DEBUG(("CACHE: DeleteStorage [%x %u]\n", record->HashNumber(), + metaData)); + + nsresult rv = NS_ERROR_UNEXPECTED; + uint32_t fileIndex = metaData ? record->MetaFile() : record->DataFile(); + nsCOMPtr<nsIFile> file; + + if (fileIndex == 0) { + // delete the file + uint32_t sizeK = metaData ? record->MetaFileSize() : record->DataFileSize(); + // XXX if sizeK == USHRT_MAX, stat file for actual size + + rv = GetFileForDiskCacheRecord(record, metaData, false, getter_AddRefs(file)); + if (NS_SUCCEEDED(rv)) { + rv = file->Remove(false); // false == non-recursive + } + DecrementTotalSize(sizeK); + + } else if (fileIndex < (kNumBlockFiles + 1)) { + // deallocate blocks + uint32_t startBlock = metaData ? record->MetaStartBlock() : record->DataStartBlock(); + uint32_t blockCount = metaData ? record->MetaBlockCount() : record->DataBlockCount(); + + rv = mBlockFile[fileIndex - 1].DeallocateBlocks(startBlock, blockCount); + DecrementTotalSize(blockCount, GetBlockSizeForIndex(fileIndex)); + } + if (metaData) record->ClearMetaLocation(); + else record->ClearDataLocation(); + + return rv; +} + + +nsresult +nsDiskCacheMap::GetFileForDiskCacheRecord(nsDiskCacheRecord * record, + bool meta, + bool createPath, + nsIFile ** result) +{ + if (!mCacheDirectory) return NS_ERROR_NOT_AVAILABLE; + + nsCOMPtr<nsIFile> file; + nsresult rv = mCacheDirectory->Clone(getter_AddRefs(file)); + if (NS_FAILED(rv)) return rv; + + uint32_t hash = record->HashNumber(); + + // The file is stored under subdirectories according to the hash number: + // 0x01234567 -> 0/12/ + rv = file->AppendNative(nsPrintfCString("%X", hash >> 28)); + if (NS_FAILED(rv)) return rv; + rv = file->AppendNative(nsPrintfCString("%02X", (hash >> 20) & 0xFF)); + if (NS_FAILED(rv)) return rv; + + bool exists; + if (createPath && (NS_FAILED(file->Exists(&exists)) || !exists)) { + rv = file->Create(nsIFile::DIRECTORY_TYPE, 0700); + if (NS_FAILED(rv)) return rv; + } + + int16_t generation = record->Generation(); + char name[32]; + // Cut the beginning of the hash that was used in the path + ::SprintfLiteral(name, "%05X%c%02X", hash & 0xFFFFF, (meta ? 'm' : 'd'), + generation); + rv = file->AppendNative(nsDependentCString(name)); + if (NS_FAILED(rv)) return rv; + + NS_IF_ADDREF(*result = file); + return rv; +} + + +nsresult +nsDiskCacheMap::GetLocalFileForDiskCacheRecord(nsDiskCacheRecord * record, + bool meta, + bool createPath, + nsIFile ** result) +{ + nsCOMPtr<nsIFile> file; + nsresult rv = GetFileForDiskCacheRecord(record, + meta, + createPath, + getter_AddRefs(file)); + if (NS_FAILED(rv)) return rv; + + NS_IF_ADDREF(*result = file); + return rv; +} + + +nsresult +nsDiskCacheMap::GetBlockFileForIndex(uint32_t index, nsIFile ** result) +{ + if (!mCacheDirectory) return NS_ERROR_NOT_AVAILABLE; + + nsCOMPtr<nsIFile> file; + nsresult rv = mCacheDirectory->Clone(getter_AddRefs(file)); + if (NS_FAILED(rv)) return rv; + + char name[32]; + ::SprintfLiteral(name, "_CACHE_%03d_", index + 1); + rv = file->AppendNative(nsDependentCString(name)); + if (NS_FAILED(rv)) return rv; + + NS_IF_ADDREF(*result = file); + + return rv; +} + + +uint32_t +nsDiskCacheMap::CalculateFileIndex(uint32_t size) +{ + // We prefer to use block file with larger block if the wasted space would + // be the same. E.g. store entry with size of 3073 bytes in 1 4K-block + // instead of in 4 1K-blocks. + + if (size <= 3 * BLOCK_SIZE_FOR_INDEX(1)) return 1; + if (size <= 3 * BLOCK_SIZE_FOR_INDEX(2)) return 2; + if (size <= 4 * BLOCK_SIZE_FOR_INDEX(3)) return 3; + return 0; +} + +nsresult +nsDiskCacheMap::EnsureBuffer(uint32_t bufSize) +{ + if (mBufferSize < bufSize) { + char * buf = (char *)PR_REALLOC(mBuffer, bufSize); + if (!buf) { + mBufferSize = 0; + return NS_ERROR_OUT_OF_MEMORY; + } + mBuffer = buf; + mBufferSize = bufSize; + } + return NS_OK; +} + +void +nsDiskCacheMap::NotifyCapacityChange(uint32_t capacity) +{ + // Heuristic 1. average cache entry size is probably around 1KB + // Heuristic 2. we don't want more than 32MB reserved to store the record + // map in memory. + const int32_t RECORD_COUNT_LIMIT = 32 * 1024 * 1024 / sizeof(nsDiskCacheRecord); + int32_t maxRecordCount = std::min(int32_t(capacity), RECORD_COUNT_LIMIT); + if (mMaxRecordCount < maxRecordCount) { + // We can only grow + mMaxRecordCount = maxRecordCount; + } +} + +size_t +nsDiskCacheMap::SizeOfExcludingThis(MallocSizeOf aMallocSizeOf) +{ + size_t usage = aMallocSizeOf(mRecordArray); + + usage += aMallocSizeOf(mBuffer); + usage += aMallocSizeOf(mMapFD); + usage += aMallocSizeOf(mCleanFD); + usage += aMallocSizeOf(mCacheDirectory); + usage += aMallocSizeOf(mCleanCacheTimer); + + for (int i = 0; i < kNumBlockFiles; i++) { + usage += mBlockFile[i].SizeOfExcludingThis(aMallocSizeOf); + } + + return usage; +} + +nsresult +nsDiskCacheMap::InitCacheClean(nsIFile * cacheDirectory, + nsDiskCache::CorruptCacheInfo * corruptInfo) +{ + // The _CACHE_CLEAN_ file will be used in the future to determine + // if the cache is clean or not. + bool cacheCleanFileExists = false; + nsCOMPtr<nsIFile> cacheCleanFile; + nsresult rv = cacheDirectory->GetParent(getter_AddRefs(cacheCleanFile)); + if (NS_SUCCEEDED(rv)) { + rv = cacheCleanFile->AppendNative( + NS_LITERAL_CSTRING("_CACHE_CLEAN_")); + if (NS_SUCCEEDED(rv)) { + // Check if the file already exists, if it does, we will later read the + // value and report it to telemetry. + cacheCleanFile->Exists(&cacheCleanFileExists); + } + } + if (NS_FAILED(rv)) { + NS_WARNING("Could not build cache clean file path"); + *corruptInfo = nsDiskCache::kCacheCleanFilePathError; + return rv; + } + + // Make sure the _CACHE_CLEAN_ file exists + rv = cacheCleanFile->OpenNSPRFileDesc(PR_RDWR | PR_CREATE_FILE, + 00600, &mCleanFD); + if (NS_FAILED(rv)) { + NS_WARNING("Could not open cache clean file"); + *corruptInfo = nsDiskCache::kCacheCleanOpenFileError; + return rv; + } + + if (cacheCleanFileExists) { + char clean = '0'; + int32_t bytesRead = PR_Read(mCleanFD, &clean, 1); + if (bytesRead != 1) { + NS_WARNING("Could not read _CACHE_CLEAN_ file contents"); + } + } + + // Create a timer that will be used to validate the cache + // as long as an activity threshold was met + mCleanCacheTimer = do_CreateInstance("@mozilla.org/timer;1", &rv); + if (NS_SUCCEEDED(rv)) { + mCleanCacheTimer->SetTarget(nsCacheService::GlobalInstance()->mCacheIOThread); + rv = ResetCacheTimer(); + } + + if (NS_FAILED(rv)) { + NS_WARNING("Could not create cache clean timer"); + mCleanCacheTimer = nullptr; + *corruptInfo = nsDiskCache::kCacheCleanTimerError; + return rv; + } + + return NS_OK; +} + +nsresult +nsDiskCacheMap::WriteCacheClean(bool clean) +{ + nsCacheService::AssertOwnsLock(); + if (!mCleanFD) { + NS_WARNING("Cache clean file is not open!"); + return NS_ERROR_FAILURE; + } + + CACHE_LOG_DEBUG(("CACHE: WriteCacheClean: %d\n", clean? 1 : 0)); + // I'm using a simple '1' or '0' to denote cache clean + // since it can be edited easily by any text editor for testing. + char data = clean? '1' : '0'; + int32_t filePos = PR_Seek(mCleanFD, 0, PR_SEEK_SET); + if (filePos != 0) { + NS_WARNING("Could not seek in cache clean file!"); + return NS_ERROR_FAILURE; + } + int32_t bytesWritten = PR_Write(mCleanFD, &data, 1); + if (bytesWritten != 1) { + NS_WARNING("Could not write cache clean file!"); + return NS_ERROR_FAILURE; + } + PRStatus err = PR_Sync(mCleanFD); + if (err != PR_SUCCESS) { + NS_WARNING("Could not flush cache clean file!"); + } + + return NS_OK; +} + +nsresult +nsDiskCacheMap::InvalidateCache() +{ + nsCacheService::AssertOwnsLock(); + CACHE_LOG_DEBUG(("CACHE: InvalidateCache\n")); + nsresult rv; + + if (!mIsDirtyCacheFlushed) { + rv = WriteCacheClean(false); + if (NS_FAILED(rv)) { + return rv; + } + + mIsDirtyCacheFlushed = true; + } + + rv = ResetCacheTimer(); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +nsresult +nsDiskCacheMap::ResetCacheTimer(int32_t timeout) +{ + mCleanCacheTimer->Cancel(); + nsresult rv = + mCleanCacheTimer->InitWithFuncCallback(RevalidateTimerCallback, + nullptr, timeout, + nsITimer::TYPE_ONE_SHOT); + NS_ENSURE_SUCCESS(rv, rv); + mLastInvalidateTime = PR_IntervalNow(); + + return rv; +} + +void +nsDiskCacheMap::RevalidateTimerCallback(nsITimer *aTimer, void *arg) +{ + nsCacheServiceAutoLock lock; + if (!nsCacheService::gService->mDiskDevice || + !nsCacheService::gService->mDiskDevice->Initialized()) { + return; + } + + nsDiskCacheMap *diskCacheMap = + &nsCacheService::gService->mDiskDevice->mCacheMap; + + // If we have less than kRevalidateCacheTimeout since the last timer was + // issued then another thread called InvalidateCache. This won't catch + // all cases where we wanted to cancel the timer, but under the lock it + // is always OK to revalidate as long as IsCacheInSafeState() returns + // true. We just want to avoid revalidating when we can to reduce IO + // and this check will do that. + uint32_t delta = + PR_IntervalToMilliseconds(PR_IntervalNow() - + diskCacheMap->mLastInvalidateTime) + + kRevalidateCacheTimeoutTolerance; + if (delta < kRevalidateCacheTimeout) { + diskCacheMap->ResetCacheTimer(); + return; + } + + nsresult rv = diskCacheMap->RevalidateCache(); + if (NS_FAILED(rv)) { + diskCacheMap->ResetCacheTimer(kRevalidateCacheErrorTimeout); + } +} + +bool +nsDiskCacheMap::IsCacheInSafeState() +{ + return nsCacheService::GlobalInstance()->IsDoomListEmpty(); +} + +nsresult +nsDiskCacheMap::RevalidateCache() +{ + CACHE_LOG_DEBUG(("CACHE: RevalidateCache\n")); + nsresult rv; + + if (!IsCacheInSafeState()) { + CACHE_LOG_DEBUG(("CACHE: Revalidation should not performed because " + "cache not in a safe state\n")); + // Normally we would return an error here, but there is a bug where + // the doom list sometimes gets an entry 'stuck' and doens't clear it + // until browser shutdown. So we allow revalidation for the time being + // to get proper telemetry data of how much the cache corruption plan + // would help. + } + + // If telemetry data shows it is worth it, we'll be flushing headers and + // records before flushing the clean cache file. + + // Write out the _CACHE_CLEAN_ file with '1' + rv = WriteCacheClean(true); + if (NS_FAILED(rv)) { + return rv; + } + + mIsDirtyCacheFlushed = false; + + return NS_OK; +} diff --git a/netwerk/cache/nsDiskCacheMap.h b/netwerk/cache/nsDiskCacheMap.h new file mode 100644 index 000000000..77af2586f --- /dev/null +++ b/netwerk/cache/nsDiskCacheMap.h @@ -0,0 +1,577 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* vim:set ts=4 sw=4 sts=4 cin et: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef _nsDiskCacheMap_h_ +#define _nsDiskCacheMap_h_ + +#include "mozilla/MemoryReporting.h" +#include <limits.h> + +#include "prnetdb.h" +#include "nsDebug.h" +#include "nsError.h" +#include "nsIFile.h" +#include "nsITimer.h" + +#include "nsDiskCache.h" +#include "nsDiskCacheBlockFile.h" + + +class nsDiskCacheBinding; +struct nsDiskCacheEntry; + +/****************************************************************************** + * nsDiskCacheRecord + * + * Cache Location Format + * + * 1000 0000 0000 0000 0000 0000 0000 0000 : initialized bit + * + * 0011 0000 0000 0000 0000 0000 0000 0000 : File Selector (0 = separate file) + * 0000 0011 0000 0000 0000 0000 0000 0000 : number of extra contiguous blocks 1-4 + * 0100 1100 0000 0000 0000 0000 0000 0000 : reserved bits + * 0000 0000 1111 1111 1111 1111 1111 1111 : block# 0-16777216 (2^24) + * + * 0000 0000 1111 1111 1111 1111 0000 0000 : eFileSizeMask (size of file in k: see note) + * 0000 0000 0000 0000 0000 0000 1111 1111 : eFileGenerationMask + * + * File Selector: + * 0 = separate file on disk + * 1 = 256 byte block file + * 2 = 1k block file + * 3 = 4k block file + * + * eFileSizeMask note: Files larger than 65535 KiB have this limit stored in + * the location. The file itself must be examined to + * determine its actual size if necessary. + * + *****************************************************************************/ + +/* + We have 3 block files with roughly the same max size (32MB) + 1 - block size 256B, number of blocks 131072 + 2 - block size 1kB, number of blocks 32768 + 3 - block size 4kB, number of blocks 8192 +*/ +#define kNumBlockFiles 3 +#define SIZE_SHIFT(idx) (2 * ((idx) - 1)) +#define BLOCK_SIZE_FOR_INDEX(idx) ((idx) ? (256 << SIZE_SHIFT(idx)) : 0) +#define BITMAP_SIZE_FOR_INDEX(idx) ((idx) ? (131072 >> SIZE_SHIFT(idx)) : 0) + +// Min and max values for the number of records in the DiskCachemap +#define kMinRecordCount 512 + +#define kSeparateFile 0 +#define kBuckets (1 << 5) // must be a power of 2! + +// Maximum size in K which can be stored in the location (see eFileSizeMask). +// Both data and metadata can be larger, but only up to kMaxDataSizeK can be +// counted into total cache size. I.e. if there are entries where either data or +// metadata is larger than kMaxDataSizeK, the total cache size will be +// inaccurate (smaller) than the actual cache size. The alternative is to stat +// the files to find the real size, which was decided against for performance +// reasons. See bug #651100 comment #21. +#define kMaxDataSizeK 0xFFFF + +// preallocate up to 1MB of separate cache file +#define kPreallocateLimit 1 * 1024 * 1024 + +// The minimum amount of milliseconds to wait before re-attempting to +// revalidate the cache. +#define kRevalidateCacheTimeout 3000 +#define kRevalidateCacheTimeoutTolerance 10 +#define kRevalidateCacheErrorTimeout 1000 + +class nsDiskCacheRecord { + +private: + uint32_t mHashNumber; + uint32_t mEvictionRank; + uint32_t mDataLocation; + uint32_t mMetaLocation; + + enum { + eLocationInitializedMask = 0x80000000, + + eLocationSelectorMask = 0x30000000, + eLocationSelectorOffset = 28, + + eExtraBlocksMask = 0x03000000, + eExtraBlocksOffset = 24, + + eReservedMask = 0x4C000000, + + eBlockNumberMask = 0x00FFFFFF, + + eFileSizeMask = 0x00FFFF00, + eFileSizeOffset = 8, + eFileGenerationMask = 0x000000FF, + eFileReservedMask = 0x4F000000 + + }; + +public: + nsDiskCacheRecord() + : mHashNumber(0), mEvictionRank(0), mDataLocation(0), mMetaLocation(0) + { + } + + bool ValidRecord() + { + if ((mDataLocation & eReservedMask) || (mMetaLocation & eReservedMask)) + return false; + return true; + } + + // HashNumber accessors + uint32_t HashNumber() const { return mHashNumber; } + void SetHashNumber( uint32_t hashNumber) { mHashNumber = hashNumber; } + + // EvictionRank accessors + uint32_t EvictionRank() const { return mEvictionRank; } + void SetEvictionRank( uint32_t rank) { mEvictionRank = rank ? rank : 1; } + + // DataLocation accessors + bool DataLocationInitialized() const { return 0 != (mDataLocation & eLocationInitializedMask); } + void ClearDataLocation() { mDataLocation = 0; } + + uint32_t DataFile() const + { + return (uint32_t)(mDataLocation & eLocationSelectorMask) >> eLocationSelectorOffset; + } + + void SetDataBlocks( uint32_t index, uint32_t startBlock, uint32_t blockCount) + { + // clear everything + mDataLocation = 0; + + // set file index + NS_ASSERTION( index < (kNumBlockFiles + 1), "invalid location index"); + NS_ASSERTION( index > 0,"invalid location index"); + mDataLocation |= (index << eLocationSelectorOffset) & eLocationSelectorMask; + + // set startBlock + NS_ASSERTION(startBlock == (startBlock & eBlockNumberMask), "invalid block number"); + mDataLocation |= startBlock & eBlockNumberMask; + + // set blockCount + NS_ASSERTION( (blockCount>=1) && (blockCount<=4),"invalid block count"); + --blockCount; + mDataLocation |= (blockCount << eExtraBlocksOffset) & eExtraBlocksMask; + + mDataLocation |= eLocationInitializedMask; + } + + uint32_t DataBlockCount() const + { + return (uint32_t)((mDataLocation & eExtraBlocksMask) >> eExtraBlocksOffset) + 1; + } + + uint32_t DataStartBlock() const + { + return (mDataLocation & eBlockNumberMask); + } + + uint32_t DataBlockSize() const + { + return BLOCK_SIZE_FOR_INDEX(DataFile()); + } + + uint32_t DataFileSize() const { return (mDataLocation & eFileSizeMask) >> eFileSizeOffset; } + void SetDataFileSize(uint32_t size) + { + NS_ASSERTION((mDataLocation & eFileReservedMask) == 0, "bad location"); + mDataLocation &= ~eFileSizeMask; // clear eFileSizeMask + mDataLocation |= (size << eFileSizeOffset) & eFileSizeMask; + } + + uint8_t DataFileGeneration() const + { + return (mDataLocation & eFileGenerationMask); + } + + void SetDataFileGeneration( uint8_t generation) + { + // clear everything, (separate file index = 0) + mDataLocation = 0; + mDataLocation |= generation & eFileGenerationMask; + mDataLocation |= eLocationInitializedMask; + } + + // MetaLocation accessors + bool MetaLocationInitialized() const { return 0 != (mMetaLocation & eLocationInitializedMask); } + void ClearMetaLocation() { mMetaLocation = 0; } + uint32_t MetaLocation() const { return mMetaLocation; } + + uint32_t MetaFile() const + { + return (uint32_t)(mMetaLocation & eLocationSelectorMask) >> eLocationSelectorOffset; + } + + void SetMetaBlocks( uint32_t index, uint32_t startBlock, uint32_t blockCount) + { + // clear everything + mMetaLocation = 0; + + // set file index + NS_ASSERTION( index < (kNumBlockFiles + 1), "invalid location index"); + NS_ASSERTION( index > 0, "invalid location index"); + mMetaLocation |= (index << eLocationSelectorOffset) & eLocationSelectorMask; + + // set startBlock + NS_ASSERTION(startBlock == (startBlock & eBlockNumberMask), "invalid block number"); + mMetaLocation |= startBlock & eBlockNumberMask; + + // set blockCount + NS_ASSERTION( (blockCount>=1) && (blockCount<=4),"invalid block count"); + --blockCount; + mMetaLocation |= (blockCount << eExtraBlocksOffset) & eExtraBlocksMask; + + mMetaLocation |= eLocationInitializedMask; + } + + uint32_t MetaBlockCount() const + { + return (uint32_t)((mMetaLocation & eExtraBlocksMask) >> eExtraBlocksOffset) + 1; + } + + uint32_t MetaStartBlock() const + { + return (mMetaLocation & eBlockNumberMask); + } + + uint32_t MetaBlockSize() const + { + return BLOCK_SIZE_FOR_INDEX(MetaFile()); + } + + uint32_t MetaFileSize() const { return (mMetaLocation & eFileSizeMask) >> eFileSizeOffset; } + void SetMetaFileSize(uint32_t size) + { + mMetaLocation &= ~eFileSizeMask; // clear eFileSizeMask + mMetaLocation |= (size << eFileSizeOffset) & eFileSizeMask; + } + + uint8_t MetaFileGeneration() const + { + return (mMetaLocation & eFileGenerationMask); + } + + void SetMetaFileGeneration( uint8_t generation) + { + // clear everything, (separate file index = 0) + mMetaLocation = 0; + mMetaLocation |= generation & eFileGenerationMask; + mMetaLocation |= eLocationInitializedMask; + } + + uint8_t Generation() const + { + if ((mDataLocation & eLocationInitializedMask) && + (DataFile() == 0)) + return DataFileGeneration(); + + if ((mMetaLocation & eLocationInitializedMask) && + (MetaFile() == 0)) + return MetaFileGeneration(); + + return 0; // no generation + } + +#if defined(IS_LITTLE_ENDIAN) + void Swap() + { + mHashNumber = htonl(mHashNumber); + mEvictionRank = htonl(mEvictionRank); + mDataLocation = htonl(mDataLocation); + mMetaLocation = htonl(mMetaLocation); + } +#endif + +#if defined(IS_LITTLE_ENDIAN) + void Unswap() + { + mHashNumber = ntohl(mHashNumber); + mEvictionRank = ntohl(mEvictionRank); + mDataLocation = ntohl(mDataLocation); + mMetaLocation = ntohl(mMetaLocation); + } +#endif + +}; + + +/****************************************************************************** + * nsDiskCacheRecordVisitor + *****************************************************************************/ + +enum { kDeleteRecordAndContinue = -1, + kStopVisitingRecords = 0, + kVisitNextRecord = 1 +}; + +class nsDiskCacheRecordVisitor { + public: + + virtual int32_t VisitRecord( nsDiskCacheRecord * mapRecord) = 0; +}; + + +/****************************************************************************** + * nsDiskCacheHeader + *****************************************************************************/ + +struct nsDiskCacheHeader { + uint32_t mVersion; // cache version. + uint32_t mDataSize; // size of cache in units of 1024bytes. + int32_t mEntryCount; // number of entries stored in cache. + uint32_t mIsDirty; // dirty flag. + int32_t mRecordCount; // Number of records + uint32_t mEvictionRank[kBuckets]; // Highest EvictionRank of the bucket + uint32_t mBucketUsage[kBuckets]; // Number of used entries in the bucket + + nsDiskCacheHeader() + : mVersion(nsDiskCache::kCurrentVersion) + , mDataSize(0) + , mEntryCount(0) + , mIsDirty(true) + , mRecordCount(0) + {} + + void Swap() + { +#if defined(IS_LITTLE_ENDIAN) + mVersion = htonl(mVersion); + mDataSize = htonl(mDataSize); + mEntryCount = htonl(mEntryCount); + mIsDirty = htonl(mIsDirty); + mRecordCount = htonl(mRecordCount); + + for (uint32_t i = 0; i < kBuckets ; i++) { + mEvictionRank[i] = htonl(mEvictionRank[i]); + mBucketUsage[i] = htonl(mBucketUsage[i]); + } +#endif + } + + void Unswap() + { +#if defined(IS_LITTLE_ENDIAN) + mVersion = ntohl(mVersion); + mDataSize = ntohl(mDataSize); + mEntryCount = ntohl(mEntryCount); + mIsDirty = ntohl(mIsDirty); + mRecordCount = ntohl(mRecordCount); + + for (uint32_t i = 0; i < kBuckets ; i++) { + mEvictionRank[i] = ntohl(mEvictionRank[i]); + mBucketUsage[i] = ntohl(mBucketUsage[i]); + } +#endif + } +}; + + +/****************************************************************************** + * nsDiskCacheMap + *****************************************************************************/ + +class nsDiskCacheMap { +public: + + nsDiskCacheMap() : + mCacheDirectory(nullptr), + mMapFD(nullptr), + mCleanFD(nullptr), + mRecordArray(nullptr), + mBufferSize(0), + mBuffer(nullptr), + mMaxRecordCount(16384), // this default value won't matter + mIsDirtyCacheFlushed(false), + mLastInvalidateTime(0) + { } + + ~nsDiskCacheMap() + { + (void) Close(true); + } + +/** + * File Operations + * + * Open + * + * Creates a new cache map file if one doesn't exist. + * Returns error if it detects change in format or cache wasn't closed. + */ + nsresult Open( nsIFile * cacheDirectory, + nsDiskCache::CorruptCacheInfo * corruptInfo); + nsresult Close(bool flush); + nsresult Trim(); + + nsresult FlushHeader(); + nsresult FlushRecords( bool unswap); + + void NotifyCapacityChange(uint32_t capacity); + +/** + * Record operations + */ + nsresult AddRecord( nsDiskCacheRecord * mapRecord, nsDiskCacheRecord * oldRecord); + nsresult UpdateRecord( nsDiskCacheRecord * mapRecord); + nsresult FindRecord( uint32_t hashNumber, nsDiskCacheRecord * mapRecord); + nsresult DeleteRecord( nsDiskCacheRecord * mapRecord); + nsresult VisitRecords( nsDiskCacheRecordVisitor * visitor); + nsresult EvictRecords( nsDiskCacheRecordVisitor * visitor); + +/** + * Disk Entry operations + */ + nsresult DeleteStorage( nsDiskCacheRecord * record); + + nsresult GetFileForDiskCacheRecord( nsDiskCacheRecord * record, + bool meta, + bool createPath, + nsIFile ** result); + + nsresult GetLocalFileForDiskCacheRecord( nsDiskCacheRecord * record, + bool meta, + bool createPath, + nsIFile ** result); + + // On success, this returns the buffer owned by nsDiskCacheMap, + // so it must not be deleted by the caller. + nsDiskCacheEntry * ReadDiskCacheEntry( nsDiskCacheRecord * record); + + nsresult WriteDiskCacheEntry( nsDiskCacheBinding * binding); + + nsresult ReadDataCacheBlocks(nsDiskCacheBinding * binding, char * buffer, uint32_t size); + nsresult WriteDataCacheBlocks(nsDiskCacheBinding * binding, char * buffer, uint32_t size); + nsresult DeleteStorage( nsDiskCacheRecord * record, bool metaData); + + /** + * Statistical Operations + */ + void IncrementTotalSize( uint32_t delta) + { + mHeader.mDataSize += delta; + mHeader.mIsDirty = true; + } + + void DecrementTotalSize( uint32_t delta) + { + NS_ASSERTION(mHeader.mDataSize >= delta, "disk cache size negative?"); + mHeader.mDataSize = mHeader.mDataSize > delta ? mHeader.mDataSize - delta : 0; + mHeader.mIsDirty = true; + } + + inline void IncrementTotalSize( uint32_t blocks, uint32_t blockSize) + { + // Round up to nearest K + IncrementTotalSize(((blocks*blockSize) + 0x03FF) >> 10); + } + + inline void DecrementTotalSize( uint32_t blocks, uint32_t blockSize) + { + // Round up to nearest K + DecrementTotalSize(((blocks*blockSize) + 0x03FF) >> 10); + } + + uint32_t TotalSize() { return mHeader.mDataSize; } + + int32_t EntryCount() { return mHeader.mEntryCount; } + + size_t SizeOfExcludingThis(mozilla::MallocSizeOf aMallocSizeOf); + + +private: + + /** + * Private methods + */ + nsresult OpenBlockFiles(nsDiskCache::CorruptCacheInfo * corruptInfo); + nsresult CloseBlockFiles(bool flush); + bool CacheFilesExist(); + + nsresult CreateCacheSubDirectories(); + + uint32_t CalculateFileIndex(uint32_t size); + + nsresult GetBlockFileForIndex( uint32_t index, nsIFile ** result); + uint32_t GetBlockSizeForIndex( uint32_t index) const { + return BLOCK_SIZE_FOR_INDEX(index); + } + uint32_t GetBitMapSizeForIndex( uint32_t index) const { + return BITMAP_SIZE_FOR_INDEX(index); + } + + // returns the bucket number + uint32_t GetBucketIndex( uint32_t hashNumber) const { + return (hashNumber & (kBuckets - 1)); + } + + // Gets the size of the bucket (in number of records) + uint32_t GetRecordsPerBucket() const { + return mHeader.mRecordCount / kBuckets; + } + + // Gets the first record in the bucket + nsDiskCacheRecord *GetFirstRecordInBucket(uint32_t bucket) const { + return mRecordArray + bucket * GetRecordsPerBucket(); + } + + uint32_t GetBucketRank(uint32_t bucketIndex, uint32_t targetRank); + + int32_t VisitEachRecord(uint32_t bucketIndex, + nsDiskCacheRecordVisitor * visitor, + uint32_t evictionRank); + + nsresult GrowRecords(); + nsresult ShrinkRecords(); + + nsresult EnsureBuffer(uint32_t bufSize); + + // The returned structure will point to the buffer owned by nsDiskCacheMap, + // so it must not be deleted by the caller. + nsDiskCacheEntry * CreateDiskCacheEntry(nsDiskCacheBinding * binding, + uint32_t * size); + + // Initializes the _CACHE_CLEAN_ related functionality + nsresult InitCacheClean(nsIFile * cacheDirectory, + nsDiskCache::CorruptCacheInfo * corruptInfo); + // Writes out a value of '0' or '1' in the _CACHE_CLEAN_ file + nsresult WriteCacheClean(bool clean); + // Resets the timout for revalidating the cache + nsresult ResetCacheTimer(int32_t timeout = kRevalidateCacheTimeout); + // Invalidates the cache, calls WriteCacheClean and ResetCacheTimer + nsresult InvalidateCache(); + // Determines if the cache is in a safe state + bool IsCacheInSafeState(); + // Revalidates the cache by writting out the header, records, and finally + // by calling WriteCacheClean(true). + nsresult RevalidateCache(); + // Timer which revalidates the cache + static void RevalidateTimerCallback(nsITimer *aTimer, void *arg); + +/** + * data members + */ +private: + nsCOMPtr<nsITimer> mCleanCacheTimer; + nsCOMPtr<nsIFile> mCacheDirectory; + PRFileDesc * mMapFD; + PRFileDesc * mCleanFD; + nsDiskCacheRecord * mRecordArray; + nsDiskCacheBlockFile mBlockFile[kNumBlockFiles]; + uint32_t mBufferSize; + char * mBuffer; + nsDiskCacheHeader mHeader; + int32_t mMaxRecordCount; + bool mIsDirtyCacheFlushed; + PRIntervalTime mLastInvalidateTime; +}; + +#endif // _nsDiskCacheMap_h_ diff --git a/netwerk/cache/nsDiskCacheStreams.cpp b/netwerk/cache/nsDiskCacheStreams.cpp new file mode 100644 index 000000000..d58cca7fd --- /dev/null +++ b/netwerk/cache/nsDiskCacheStreams.cpp @@ -0,0 +1,693 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + + +#include "nsCache.h" +#include "nsDiskCache.h" +#include "nsDiskCacheDevice.h" +#include "nsDiskCacheStreams.h" +#include "nsCacheService.h" +#include "mozilla/FileUtils.h" +#include "nsThreadUtils.h" +#include "mozilla/MemoryReporting.h" +#include "mozilla/Telemetry.h" +#include "mozilla/TimeStamp.h" +#include <algorithm> + +// we pick 16k as the max buffer size because that is the threshold above which +// we are unable to store the data in the cache block files +// see nsDiskCacheMap.[cpp,h] +#define kMaxBufferSize (16 * 1024) + +// Assumptions: +// - cache descriptors live for life of streams +// - streams will only be used by FileTransport, +// they will not be directly accessible to clients +// - overlapped I/O is NOT supported + + +/****************************************************************************** + * nsDiskCacheInputStream + *****************************************************************************/ +class nsDiskCacheInputStream : public nsIInputStream { + +public: + + nsDiskCacheInputStream( nsDiskCacheStreamIO * parent, + PRFileDesc * fileDesc, + const char * buffer, + uint32_t endOfStream); + + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSIINPUTSTREAM + +private: + virtual ~nsDiskCacheInputStream(); + + nsDiskCacheStreamIO * mStreamIO; // backpointer to parent + PRFileDesc * mFD; + const char * mBuffer; + uint32_t mStreamEnd; + uint32_t mPos; // stream position + bool mClosed; +}; + + +NS_IMPL_ISUPPORTS(nsDiskCacheInputStream, nsIInputStream) + + +nsDiskCacheInputStream::nsDiskCacheInputStream( nsDiskCacheStreamIO * parent, + PRFileDesc * fileDesc, + const char * buffer, + uint32_t endOfStream) + : mStreamIO(parent) + , mFD(fileDesc) + , mBuffer(buffer) + , mStreamEnd(endOfStream) + , mPos(0) + , mClosed(false) +{ + NS_ADDREF(mStreamIO); + mStreamIO->IncrementInputStreamCount(); +} + + +nsDiskCacheInputStream::~nsDiskCacheInputStream() +{ + Close(); + mStreamIO->DecrementInputStreamCount(); + NS_RELEASE(mStreamIO); +} + + +NS_IMETHODIMP +nsDiskCacheInputStream::Close() +{ + if (!mClosed) { + if (mFD) { + (void) PR_Close(mFD); + mFD = nullptr; + } + mClosed = true; + } + return NS_OK; +} + + +NS_IMETHODIMP +nsDiskCacheInputStream::Available(uint64_t * bytesAvailable) +{ + if (mClosed) return NS_BASE_STREAM_CLOSED; + if (mStreamEnd < mPos) return NS_ERROR_UNEXPECTED; + + *bytesAvailable = mStreamEnd - mPos; + return NS_OK; +} + + +NS_IMETHODIMP +nsDiskCacheInputStream::Read(char * buffer, uint32_t count, uint32_t * bytesRead) +{ + *bytesRead = 0; + + if (mClosed) { + CACHE_LOG_DEBUG(("CACHE: nsDiskCacheInputStream::Read " + "[stream=%p] stream was closed", + this, buffer, count)); + return NS_OK; + } + + if (mPos == mStreamEnd) { + CACHE_LOG_DEBUG(("CACHE: nsDiskCacheInputStream::Read " + "[stream=%p] stream at end of file", + this, buffer, count)); + return NS_OK; + } + if (mPos > mStreamEnd) { + CACHE_LOG_DEBUG(("CACHE: nsDiskCacheInputStream::Read " + "[stream=%p] stream past end of file (!)", + this, buffer, count)); + return NS_ERROR_UNEXPECTED; + } + + if (count > mStreamEnd - mPos) + count = mStreamEnd - mPos; + + if (mFD) { + // just read from file + int32_t result = PR_Read(mFD, buffer, count); + if (result < 0) { + nsresult rv = NS_ErrorAccordingToNSPR(); + CACHE_LOG_DEBUG(("CACHE: nsDiskCacheInputStream::Read PR_Read failed" + "[stream=%p, rv=%d, NSPR error %s", + this, int(rv), PR_ErrorToName(PR_GetError()))); + return rv; + } + + mPos += (uint32_t)result; + *bytesRead = (uint32_t)result; + + } else if (mBuffer) { + // read data from mBuffer + memcpy(buffer, mBuffer + mPos, count); + mPos += count; + *bytesRead = count; + } else { + // no data source for input stream + } + + CACHE_LOG_DEBUG(("CACHE: nsDiskCacheInputStream::Read " + "[stream=%p, count=%ud, byteRead=%ud] ", + this, unsigned(count), unsigned(*bytesRead))); + return NS_OK; +} + + +NS_IMETHODIMP +nsDiskCacheInputStream::ReadSegments(nsWriteSegmentFun writer, + void * closure, + uint32_t count, + uint32_t * bytesRead) +{ + return NS_ERROR_NOT_IMPLEMENTED; +} + + +NS_IMETHODIMP +nsDiskCacheInputStream::IsNonBlocking(bool * nonBlocking) +{ + *nonBlocking = false; + return NS_OK; +} + + + + +/****************************************************************************** + * nsDiskCacheStreamIO + *****************************************************************************/ +NS_IMPL_ISUPPORTS(nsDiskCacheStreamIO, nsIOutputStream) + +nsDiskCacheStreamIO::nsDiskCacheStreamIO(nsDiskCacheBinding * binding) + : mBinding(binding) + , mInStreamCount(0) + , mFD(nullptr) + , mStreamEnd(0) + , mBufSize(0) + , mBuffer(nullptr) + , mOutputStreamIsOpen(false) +{ + mDevice = (nsDiskCacheDevice *)mBinding->mCacheEntry->CacheDevice(); + + // acquire "death grip" on cache service + nsCacheService *service = nsCacheService::GlobalInstance(); + NS_ADDREF(service); +} + + +nsDiskCacheStreamIO::~nsDiskCacheStreamIO() +{ + nsCacheService::AssertOwnsLock(); + + // Close the outputstream + if (mBinding && mOutputStreamIsOpen) { + (void)CloseOutputStream(); + } + + // release "death grip" on cache service + nsCacheService *service = nsCacheService::GlobalInstance(); + NS_RELEASE(service); + + // assert streams closed + NS_ASSERTION(!mOutputStreamIsOpen, "output stream still open"); + NS_ASSERTION(mInStreamCount == 0, "input stream still open"); + NS_ASSERTION(!mFD, "file descriptor not closed"); + + DeleteBuffer(); +} + + +// NOTE: called with service lock held +nsresult +nsDiskCacheStreamIO::GetInputStream(uint32_t offset, nsIInputStream ** inputStream) +{ + NS_ENSURE_ARG_POINTER(inputStream); + NS_ENSURE_TRUE(offset == 0, NS_ERROR_NOT_IMPLEMENTED); + + *inputStream = nullptr; + + if (!mBinding) return NS_ERROR_NOT_AVAILABLE; + + if (mOutputStreamIsOpen) { + NS_WARNING("already have an output stream open"); + return NS_ERROR_NOT_AVAILABLE; + } + + nsresult rv; + PRFileDesc * fd = nullptr; + + mStreamEnd = mBinding->mCacheEntry->DataSize(); + if (mStreamEnd == 0) { + // there's no data to read + NS_ASSERTION(!mBinding->mRecord.DataLocationInitialized(), "storage allocated for zero data size"); + } else if (mBinding->mRecord.DataFile() == 0) { + // open file desc for data + rv = OpenCacheFile(PR_RDONLY, &fd); + if (NS_FAILED(rv)) return rv; // unable to open file + NS_ASSERTION(fd, "cache stream lacking open file."); + + } else if (!mBuffer) { + // read block file for data + rv = ReadCacheBlocks(mStreamEnd); + if (NS_FAILED(rv)) return rv; + } + // else, mBuffer already contains all of the data (left over from a + // previous block-file read or write). + + NS_ASSERTION(!(fd && mBuffer), "ambiguous data sources for input stream"); + + // create a new input stream + nsDiskCacheInputStream * inStream = new nsDiskCacheInputStream(this, fd, mBuffer, mStreamEnd); + if (!inStream) return NS_ERROR_OUT_OF_MEMORY; + + NS_ADDREF(*inputStream = inStream); + return NS_OK; +} + + +// NOTE: called with service lock held +nsresult +nsDiskCacheStreamIO::GetOutputStream(uint32_t offset, nsIOutputStream ** outputStream) +{ + NS_ENSURE_ARG_POINTER(outputStream); + *outputStream = nullptr; + + if (!mBinding) return NS_ERROR_NOT_AVAILABLE; + + NS_ASSERTION(!mOutputStreamIsOpen, "already have an output stream open"); + NS_ASSERTION(mInStreamCount == 0, "we already have input streams open"); + if (mOutputStreamIsOpen || mInStreamCount) return NS_ERROR_NOT_AVAILABLE; + + mStreamEnd = mBinding->mCacheEntry->DataSize(); + + // Inits file or buffer and truncate at the desired offset + nsresult rv = SeekAndTruncate(offset); + if (NS_FAILED(rv)) return rv; + + mOutputStreamIsOpen = true; + NS_ADDREF(*outputStream = this); + return NS_OK; +} + +nsresult +nsDiskCacheStreamIO::ClearBinding() +{ + nsresult rv = NS_OK; + if (mBinding && mOutputStreamIsOpen) + rv = CloseOutputStream(); + mBinding = nullptr; + return rv; +} + +NS_IMETHODIMP +nsDiskCacheStreamIO::Close() +{ + if (!mOutputStreamIsOpen) return NS_OK; + + // grab service lock + nsCacheServiceAutoLock lock; + + if (!mBinding) { // if we're severed, just clear member variables + mOutputStreamIsOpen = false; + return NS_ERROR_NOT_AVAILABLE; + } + + nsresult rv = CloseOutputStream(); + if (NS_FAILED(rv)) + NS_WARNING("CloseOutputStream() failed"); + + return rv; +} + +nsresult +nsDiskCacheStreamIO::CloseOutputStream() +{ + NS_ASSERTION(mBinding, "oops"); + + CACHE_LOG_DEBUG(("CACHE: CloseOutputStream [%x doomed=%u]\n", + mBinding->mRecord.HashNumber(), mBinding->mDoomed)); + + // Mark outputstream as closed, even if saving the stream fails + mOutputStreamIsOpen = false; + + // When writing to a file, just close the file + if (mFD) { + (void) PR_Close(mFD); + mFD = nullptr; + return NS_OK; + } + + // write data to cache blocks, or flush mBuffer to file + NS_ASSERTION(mStreamEnd <= kMaxBufferSize, "stream is bigger than buffer"); + + nsDiskCacheMap *cacheMap = mDevice->CacheMap(); // get map reference + nsDiskCacheRecord * record = &mBinding->mRecord; + nsresult rv = NS_OK; + + // delete existing storage + if (record->DataLocationInitialized()) { + rv = cacheMap->DeleteStorage(record, nsDiskCache::kData); + NS_ENSURE_SUCCESS(rv, rv); + + // Only call UpdateRecord when there is no data to write, + // because WriteDataCacheBlocks / FlushBufferToFile calls it. + if ((mStreamEnd == 0) && (!mBinding->mDoomed)) { + rv = cacheMap->UpdateRecord(record); + if (NS_FAILED(rv)) { + NS_WARNING("cacheMap->UpdateRecord() failed."); + return rv; // XXX doom cache entry + } + } + } + + if (mStreamEnd == 0) return NS_OK; // nothing to write + + // try to write to the cache blocks + rv = cacheMap->WriteDataCacheBlocks(mBinding, mBuffer, mStreamEnd); + if (NS_FAILED(rv)) { + NS_WARNING("WriteDataCacheBlocks() failed."); + + // failed to store in cacheblocks, save as separate file + rv = FlushBufferToFile(); // initializes DataFileLocation() if necessary + if (mFD) { + UpdateFileSize(); + (void) PR_Close(mFD); + mFD = nullptr; + } + else + NS_WARNING("no file descriptor"); + } + + return rv; +} + + +// assumptions: +// only one thread writing at a time +// never have both output and input streams open +// OnDataSizeChanged() will have already been called to update entry->DataSize() + +NS_IMETHODIMP +nsDiskCacheStreamIO::Write( const char * buffer, + uint32_t count, + uint32_t * bytesWritten) +{ + NS_ENSURE_ARG_POINTER(buffer); + NS_ENSURE_ARG_POINTER(bytesWritten); + if (!mOutputStreamIsOpen) return NS_BASE_STREAM_CLOSED; + + *bytesWritten = 0; // always initialize to zero in case of errors + + NS_ASSERTION(count, "Write called with count of zero"); + if (count == 0) { + return NS_OK; // nothing to write + } + + // grab service lock + nsCacheServiceAutoLock lock; + if (!mBinding) return NS_ERROR_NOT_AVAILABLE; + + if (mInStreamCount) { + // we have open input streams already + // this is an error until we support overlapped I/O + NS_WARNING("Attempting to write to cache entry with open input streams.\n"); + return NS_ERROR_NOT_AVAILABLE; + } + + // Not writing to file, and it will fit in the cachedatablocks? + if (!mFD && (mStreamEnd + count <= kMaxBufferSize)) { + + // We have more data than the current buffer size? + if ((mStreamEnd + count > mBufSize) && (mBufSize < kMaxBufferSize)) { + // Increase buffer to the maximum size. + mBuffer = (char *) moz_xrealloc(mBuffer, kMaxBufferSize); + mBufSize = kMaxBufferSize; + } + + // Store in the buffer but only if it fits + if (mStreamEnd + count <= mBufSize) { + memcpy(mBuffer + mStreamEnd, buffer, count); + mStreamEnd += count; + *bytesWritten = count; + return NS_OK; + } + } + + // There are more bytes than fit in the buffer/cacheblocks, switch to file + if (!mFD) { + // Opens a cache file and write the buffer to it + nsresult rv = FlushBufferToFile(); + if (NS_FAILED(rv)) { + return rv; + } + } + // Write directly to the file + if (PR_Write(mFD, buffer, count) != (int32_t)count) { + NS_WARNING("failed to write all data"); + return NS_ERROR_UNEXPECTED; // NS_ErrorAccordingToNSPR() + } + mStreamEnd += count; + *bytesWritten = count; + + UpdateFileSize(); + NS_ASSERTION(mBinding->mCacheEntry->DataSize() == mStreamEnd, "bad stream"); + + return NS_OK; +} + + +void +nsDiskCacheStreamIO::UpdateFileSize() +{ + NS_ASSERTION(mFD, "nsDiskCacheStreamIO::UpdateFileSize should not have been called"); + + nsDiskCacheRecord * record = &mBinding->mRecord; + const uint32_t oldSizeK = record->DataFileSize(); + uint32_t newSizeK = (mStreamEnd + 0x03FF) >> 10; + + // make sure the size won't overflow (bug #651100) + if (newSizeK > kMaxDataSizeK) + newSizeK = kMaxDataSizeK; + + if (newSizeK == oldSizeK) return; + + record->SetDataFileSize(newSizeK); + + // update cache size totals + nsDiskCacheMap * cacheMap = mDevice->CacheMap(); + cacheMap->DecrementTotalSize(oldSizeK); // decrement old size + cacheMap->IncrementTotalSize(newSizeK); // increment new size + + if (!mBinding->mDoomed) { + nsresult rv = cacheMap->UpdateRecord(record); + if (NS_FAILED(rv)) { + NS_WARNING("cacheMap->UpdateRecord() failed."); + // XXX doom cache entry? + } + } +} + + +nsresult +nsDiskCacheStreamIO::OpenCacheFile(int flags, PRFileDesc ** fd) +{ + NS_ENSURE_ARG_POINTER(fd); + + CACHE_LOG_DEBUG(("nsDiskCacheStreamIO::OpenCacheFile")); + + nsresult rv; + nsDiskCacheMap * cacheMap = mDevice->CacheMap(); + nsCOMPtr<nsIFile> localFile; + + rv = cacheMap->GetLocalFileForDiskCacheRecord(&mBinding->mRecord, + nsDiskCache::kData, + !!(flags & PR_CREATE_FILE), + getter_AddRefs(localFile)); + if (NS_FAILED(rv)) return rv; + + // create PRFileDesc for input stream - the 00600 is just for consistency + return localFile->OpenNSPRFileDesc(flags, 00600, fd); +} + + +nsresult +nsDiskCacheStreamIO::ReadCacheBlocks(uint32_t bufferSize) +{ + NS_ASSERTION(mStreamEnd == mBinding->mCacheEntry->DataSize(), "bad stream"); + NS_ASSERTION(bufferSize <= kMaxBufferSize, "bufferSize too large for buffer"); + NS_ASSERTION(mStreamEnd <= bufferSize, "data too large for buffer"); + + nsDiskCacheRecord * record = &mBinding->mRecord; + if (!record->DataLocationInitialized()) return NS_OK; + + NS_ASSERTION(record->DataFile() != kSeparateFile, "attempt to read cache blocks on separate file"); + + if (!mBuffer) { + mBuffer = (char *) moz_xmalloc(bufferSize); + mBufSize = bufferSize; + } + + // read data stored in cache block files + nsDiskCacheMap *map = mDevice->CacheMap(); // get map reference + return map->ReadDataCacheBlocks(mBinding, mBuffer, mStreamEnd); +} + + +nsresult +nsDiskCacheStreamIO::FlushBufferToFile() +{ + nsresult rv; + nsDiskCacheRecord * record = &mBinding->mRecord; + + if (!mFD) { + if (record->DataLocationInitialized() && (record->DataFile() > 0)) { + // remove cache block storage + nsDiskCacheMap * cacheMap = mDevice->CacheMap(); + rv = cacheMap->DeleteStorage(record, nsDiskCache::kData); + if (NS_FAILED(rv)) return rv; + } + record->SetDataFileGeneration(mBinding->mGeneration); + + // allocate file + rv = OpenCacheFile(PR_RDWR | PR_CREATE_FILE, &mFD); + if (NS_FAILED(rv)) return rv; + + int64_t dataSize = mBinding->mCacheEntry->PredictedDataSize(); + if (dataSize != -1) + mozilla::fallocate(mFD, std::min<int64_t>(dataSize, kPreallocateLimit)); + } + + // write buffer to the file when there is data in it + if (mStreamEnd > 0) { + if (!mBuffer) { + NS_RUNTIMEABORT("Fix me!"); + } + if (PR_Write(mFD, mBuffer, mStreamEnd) != (int32_t)mStreamEnd) { + NS_WARNING("failed to flush all data"); + return NS_ERROR_UNEXPECTED; // NS_ErrorAccordingToNSPR() + } + } + + // buffer is no longer valid + DeleteBuffer(); + + return NS_OK; +} + + +void +nsDiskCacheStreamIO::DeleteBuffer() +{ + if (mBuffer) { + free(mBuffer); + mBuffer = nullptr; + mBufSize = 0; + } +} + +size_t +nsDiskCacheStreamIO::SizeOfIncludingThis(mozilla::MallocSizeOf aMallocSizeOf) +{ + size_t usage = aMallocSizeOf(this); + + usage += aMallocSizeOf(mFD); + usage += aMallocSizeOf(mBuffer); + + return usage; +} + +nsresult +nsDiskCacheStreamIO::SeekAndTruncate(uint32_t offset) +{ + if (!mBinding) return NS_ERROR_NOT_AVAILABLE; + + if (uint32_t(offset) > mStreamEnd) return NS_ERROR_FAILURE; + + // Set the current end to the desired offset + mStreamEnd = offset; + + // Currently stored in file? + if (mBinding->mRecord.DataLocationInitialized() && + (mBinding->mRecord.DataFile() == 0)) { + if (!mFD) { + // we need an mFD, we better open it now + nsresult rv = OpenCacheFile(PR_RDWR | PR_CREATE_FILE, &mFD); + if (NS_FAILED(rv)) return rv; + } + if (offset) { + if (PR_Seek(mFD, offset, PR_SEEK_SET) == -1) + return NS_ErrorAccordingToNSPR(); + } + nsDiskCache::Truncate(mFD, offset); + UpdateFileSize(); + + // When we starting at zero again, close file and start with buffer. + // If offset is non-zero (and within buffer) an option would be + // to read the file into the buffer, but chance is high that it is + // rewritten to the file anyway. + if (offset == 0) { + // close file descriptor + (void) PR_Close(mFD); + mFD = nullptr; + } + return NS_OK; + } + + // read data into mBuffer if not read yet. + if (offset && !mBuffer) { + nsresult rv = ReadCacheBlocks(kMaxBufferSize); + if (NS_FAILED(rv)) return rv; + } + + // stream buffer sanity check + NS_ASSERTION(mStreamEnd <= kMaxBufferSize, "bad stream"); + return NS_OK; +} + + +NS_IMETHODIMP +nsDiskCacheStreamIO::Flush() +{ + if (!mOutputStreamIsOpen) return NS_BASE_STREAM_CLOSED; + return NS_OK; +} + + +NS_IMETHODIMP +nsDiskCacheStreamIO::WriteFrom(nsIInputStream *inStream, uint32_t count, uint32_t *bytesWritten) +{ + NS_NOTREACHED("WriteFrom"); + return NS_ERROR_NOT_IMPLEMENTED; +} + + +NS_IMETHODIMP +nsDiskCacheStreamIO::WriteSegments( nsReadSegmentFun reader, + void * closure, + uint32_t count, + uint32_t * bytesWritten) +{ + NS_NOTREACHED("WriteSegments"); + return NS_ERROR_NOT_IMPLEMENTED; +} + + +NS_IMETHODIMP +nsDiskCacheStreamIO::IsNonBlocking(bool * nonBlocking) +{ + *nonBlocking = false; + return NS_OK; +} diff --git a/netwerk/cache/nsDiskCacheStreams.h b/netwerk/cache/nsDiskCacheStreams.h new file mode 100644 index 000000000..247c16a6e --- /dev/null +++ b/netwerk/cache/nsDiskCacheStreams.h @@ -0,0 +1,70 @@ +/* -*- Mode: C++; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + + +#ifndef _nsDiskCacheStreams_h_ +#define _nsDiskCacheStreams_h_ + +#include "mozilla/MemoryReporting.h" +#include "nsDiskCacheBinding.h" + +#include "nsCache.h" + +#include "nsIInputStream.h" +#include "nsIOutputStream.h" + +#include "mozilla/Atomics.h" + +class nsDiskCacheDevice; + +class nsDiskCacheStreamIO : public nsIOutputStream { +public: + explicit nsDiskCacheStreamIO(nsDiskCacheBinding * binding); + + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSIOUTPUTSTREAM + + nsresult GetInputStream(uint32_t offset, nsIInputStream ** inputStream); + nsresult GetOutputStream(uint32_t offset, nsIOutputStream ** outputStream); + + nsresult ClearBinding(); + + void IncrementInputStreamCount() { mInStreamCount++; } + void DecrementInputStreamCount() + { + mInStreamCount--; + NS_ASSERTION(mInStreamCount >= 0, "mInStreamCount has gone negative"); + } + + size_t SizeOfIncludingThis(mozilla::MallocSizeOf aMallocSizeOf); + + // GCC 2.95.2 requires this to be defined, although we never call it. + // and OS/2 requires that it not be private + nsDiskCacheStreamIO() { NS_NOTREACHED("oops"); } + +private: + virtual ~nsDiskCacheStreamIO(); + + nsresult OpenCacheFile(int flags, PRFileDesc ** fd); + nsresult ReadCacheBlocks(uint32_t bufferSize); + nsresult FlushBufferToFile(); + void UpdateFileSize(); + void DeleteBuffer(); + nsresult CloseOutputStream(); + nsresult SeekAndTruncate(uint32_t offset); + + nsDiskCacheBinding * mBinding; // not an owning reference + nsDiskCacheDevice * mDevice; + mozilla::Atomic<int32_t> mInStreamCount; + PRFileDesc * mFD; + + uint32_t mStreamEnd; // current size of data + uint32_t mBufSize; // current end of buffer + char * mBuffer; + bool mOutputStreamIsOpen; +}; + +#endif // _nsDiskCacheStreams_h_ diff --git a/netwerk/cache/nsICache.idl b/netwerk/cache/nsICache.idl new file mode 100644 index 000000000..95bab1515 --- /dev/null +++ b/netwerk/cache/nsICache.idl @@ -0,0 +1,142 @@ +/* -*- Mode: IDL; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +typedef long nsCacheStoragePolicy; +typedef long nsCacheAccessMode; + +/** + * nsICache is a namespace for various cache constants. It does not represent + * an actual object. + */ +[scriptable, uuid(d6c67f38-b39a-4582-8a48-4c4f8a56dfd0)] +interface nsICache +{ + /** + * Access Modes + * + * + * Mode Requested | Not Cached | Cached + * ------------------------------------------------------------------------ + * READ | KEY_NOT_FOUND | NS_OK + * | Mode = NONE | Mode = READ + * | No Descriptor | Descriptor + * ------------------------------------------------------------------------ + * WRITE | NS_OK | NS_OK (Cache service + * | Mode = WRITE | Mode = WRITE dooms existing + * | Descriptor | Descriptor cache entry) + * ------------------------------------------------------------------------ + * READ_WRITE | NS_OK | NS_OK + * (1st req.) | Mode = WRITE | Mode = READ_WRITE + * | Descriptor | Descriptor + * ------------------------------------------------------------------------ + * READ_WRITE | N/A | NS_OK + * (Nth req.) | | Mode = READ + * | | Descriptor + * ------------------------------------------------------------------------ + * + * + * Access Requested: + * + * READ - I only want to READ, if there isn't an entry just fail + * WRITE - I have something new I want to write into the cache, make + * me a new entry and doom the old one, if any. + * READ_WRITE - I want to READ, but I'm willing to update an existing + * entry if necessary, or create a new one if none exists. + * + * + * Access Granted: + * + * NONE - No descriptor is provided. You get zilch. Nada. Nothing. + * READ - You can READ from this descriptor. + * WRITE - You must WRITE to this descriptor because the cache entry + * was just created for you. + * READ_WRITE - You can READ the descriptor to determine if it's valid, + * you may WRITE if it needs updating. + * + * + * Comments: + * + * If you think that you might need to modify cached data or meta data, + * then you must open a cache entry requesting WRITE access. Only one + * cache entry descriptor, per cache entry, will be granted WRITE access. + * + * Usually, you will request READ_WRITE access in order to first test the + * meta data and informational fields to determine if a write (ie. going + * to the net) may actually be necessary. If you determine that it is + * not, then you would mark the cache entry as valid (using MarkValid) and + * then simply read the data from the cache. + * + * A descriptor granted WRITE access has exclusive access to the cache + * entry up to the point at which it marks it as valid. Once the cache + * entry has been "validated", other descriptors with READ access may be + * opened to the cache entry. + * + * If you make a request for READ_WRITE access to a cache entry, the cache + * service will downgrade your access to READ if there is already a + * cache entry descriptor open with WRITE access. + * + * If you make a request for only WRITE access to a cache entry and another + * descriptor with WRITE access is currently open, then the existing cache + * entry will be 'doomed', and you will be given a descriptor (with WRITE + * access only) to a new cache entry. + * + */ + const nsCacheAccessMode ACCESS_NONE = 0; + const nsCacheAccessMode ACCESS_READ = 1; + const nsCacheAccessMode ACCESS_WRITE = 2; + const nsCacheAccessMode ACCESS_READ_WRITE = 3; + + /** + * Storage Policy + * + * The storage policy of a cache entry determines the device(s) to which + * it belongs. See nsICacheSession and nsICacheEntryDescriptor for more + * details. + * + * STORE_ANYWHERE - Allows the cache entry to be stored in any device. + * The cache service decides which cache device to use + * based on "some resource management calculation." + * STORE_IN_MEMORY - Requires the cache entry to reside in non-persistent + * storage (ie. typically in system RAM). + * STORE_ON_DISK - Requires the cache entry to reside in persistent + * storage (ie. typically on a system's hard disk). + * STORE_OFFLINE - Requires the cache entry to reside in persistent, + * reliable storage for offline use. + */ + const nsCacheStoragePolicy STORE_ANYWHERE = 0; + const nsCacheStoragePolicy STORE_IN_MEMORY = 1; + const nsCacheStoragePolicy STORE_ON_DISK = 2; + // value 3 was used by STORE_ON_DISK_AS_FILE which was removed + const nsCacheStoragePolicy STORE_OFFLINE = 4; + + /** + * All entries for a cache session are stored as streams of data or + * as objects. These constant my be used to specify the type of entries + * when calling nsICacheService::CreateSession(). + */ + const long NOT_STREAM_BASED = 0; + const long STREAM_BASED = 1; + + /** + * The synchronous OpenCacheEntry() may be blocking or non-blocking. If a cache entry is + * waiting to be validated by another cache descriptor (so no new cache descriptors for that + * key can be created, OpenCacheEntry() will return NS_ERROR_CACHE_WAIT_FOR_VALIDATION in + * non-blocking mode. In blocking mode, it will wait until the cache entry for the key has + * been validated or doomed. If the cache entry is validated, then a descriptor for that + * entry will be created and returned. If the cache entry was doomed, then a descriptor + * will be created for a new cache entry for the key. + */ + const long NON_BLOCKING = 0; + const long BLOCKING = 1; + + /** + * Constant meaning no expiration time. + */ + const unsigned long NO_EXPIRATION_TIME = 0xFFFFFFFF; +}; + diff --git a/netwerk/cache/nsICacheEntryDescriptor.idl b/netwerk/cache/nsICacheEntryDescriptor.idl new file mode 100644 index 000000000..20fac747d --- /dev/null +++ b/netwerk/cache/nsICacheEntryDescriptor.idl @@ -0,0 +1,164 @@ +/* -*- Mode: IDL; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsICacheVisitor.idl" +#include "nsICache.idl" + +interface nsISimpleEnumerator; +interface nsICacheListener; +interface nsIInputStream; +interface nsIOutputStream; +interface nsIFile; +interface nsICacheMetaDataVisitor; + + +[scriptable, uuid(90b17d31-46aa-4fb1-a206-473c966cbc18)] +interface nsICacheEntryDescriptor : nsICacheEntryInfo +{ + /** + * Set the time at which the cache entry should be considered invalid (in + * seconds since the Epoch). + */ + void setExpirationTime(in uint32_t expirationTime); + + /** + * Set the cache entry data size. This will fail if the cache entry + * IS stream based. + */ + void setDataSize(in unsigned long size); + + /** + * Open blocking input stream to cache data. This will fail if the cache + * entry IS NOT stream based. Use the stream transport service to + * asynchronously read this stream on a background thread. The returned + * stream MAY implement nsISeekableStream. + * + * @param offset + * read starting from this offset into the cached data. an offset + * beyond the end of the stream has undefined consequences. + * + * @return blocking, unbuffered input stream. + */ + nsIInputStream openInputStream(in unsigned long offset); + + /** + * Open blocking output stream to cache data. This will fail if the cache + * entry IS NOT stream based. Use the stream transport service to + * asynchronously write to this stream on a background thread. The returned + * stream MAY implement nsISeekableStream. + * + * If opening an output stream to existing cached data, the data will be + * truncated to the specified offset. + * + * @param offset + * write starting from this offset into the cached data. an offset + * beyond the end of the stream has undefined consequences. + * + * @return blocking, unbuffered output stream. + */ + nsIOutputStream openOutputStream(in unsigned long offset); + + /** + * Get/set the cache data element. This will fail if the cache entry + * IS stream based. The cache entry holds a strong reference to this + * object. The object will be released when the cache entry is destroyed. + */ + attribute nsISupports cacheElement; + + /** + * Stores the Content-Length specified in the HTTP header for this + * entry. Checked before we write to the cache entry, to prevent ever + * taking up space in the cache for an entry that we know up front + * is going to have to be evicted anyway. See bug 588507. + */ + attribute int64_t predictedDataSize; + + /** + * Get the access granted to this descriptor. See nsICache.idl for the + * definitions of the access modes and a thorough description of their + * corresponding meanings. + */ + readonly attribute nsCacheAccessMode accessGranted; + + /** + * Get/set the storage policy of the cache entry. See nsICache.idl for + * the definitions of the storage policies. + */ + attribute nsCacheStoragePolicy storagePolicy; + + /** + * Get the disk file associated with the cache entry. + */ + readonly attribute nsIFile file; + + /** + * Get/set security info on the cache entry for this descriptor. This fails + * if the storage policy is not STORE_IN_MEMORY. + */ + attribute nsISupports securityInfo; + + /** + * Get the size of the cache entry data, as stored. This may differ + * from the entry's dataSize, if the entry is compressed. + */ + readonly attribute unsigned long storageDataSize; + + /** + * Doom the cache entry this descriptor references in order to slate it for + * removal. Once doomed a cache entry cannot be undoomed. + * + * A descriptor with WRITE access can doom the cache entry and choose to + * fail pending requests. This means that pending requests will not get + * a cache descriptor. This is meant as a tool for clients that wish to + * instruct pending requests to skip the cache. + */ + void doom(); + void doomAndFailPendingRequests(in nsresult status); + + /** + * Asynchronously doom an entry. Listener will be notified about the status + * of the operation. Null may be passed if caller doesn't care about the + * result. + */ + void asyncDoom(in nsICacheListener listener); + + /** + * A writer must validate this cache object before any readers are given + * a descriptor to the object. + */ + void markValid(); + + /** + * Explicitly close the descriptor (optional). + */ + + void close(); + + /** + * Methods for accessing meta data. Meta data is a table of key/value + * string pairs. The strings do not have to conform to any particular + * charset, but they must be null terminated. + */ + string getMetaDataElement(in string key); + void setMetaDataElement(in string key, in string value); + + /** + * Visitor will be called with key/value pair for each meta data element. + */ + void visitMetaData(in nsICacheMetaDataVisitor visitor); +}; + + + +[scriptable, uuid(22f9a49c-3cf8-4c23-8006-54efb11ac562)] +interface nsICacheMetaDataVisitor : nsISupports +{ + /** + * Called for each key/value pair in the meta data for a cache entry + */ + boolean visitMetaDataElement(in string key, + in string value); +}; diff --git a/netwerk/cache/nsICacheListener.idl b/netwerk/cache/nsICacheListener.idl new file mode 100644 index 000000000..cfb2dd470 --- /dev/null +++ b/netwerk/cache/nsICacheListener.idl @@ -0,0 +1,32 @@ +/* -*- Mode: IDL; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + + +#include "nsISupports.idl" +#include "nsICache.idl" + + +interface nsICacheEntryDescriptor; + +[scriptable, uuid(8eadf2ed-8cac-4961-8025-6da6d5827e74)] +interface nsICacheListener : nsISupports +{ + /** + * Called when the requested access (or appropriate subset) is + * acquired. The status parameter equals NS_OK on success. + * See nsICacheService.idl for accessGranted values. + */ + void onCacheEntryAvailable(in nsICacheEntryDescriptor descriptor, + in nsCacheAccessMode accessGranted, + in nsresult status); + + /** + * Called when nsCacheSession::DoomEntry() is completed. The status + * parameter is NS_OK when the entry was doomed, or NS_ERROR_NOT_AVAILABLE + * when there is no such entry. + */ + void onCacheEntryDoomed(in nsresult status); +}; diff --git a/netwerk/cache/nsICacheService.idl b/netwerk/cache/nsICacheService.idl new file mode 100644 index 000000000..b015a7244 --- /dev/null +++ b/netwerk/cache/nsICacheService.idl @@ -0,0 +1,98 @@ +/* -*- Mode: IDL; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" +#include "nsICache.idl" + +interface nsISimpleEnumerator; +interface nsICacheListener; +interface nsICacheSession; +interface nsICacheVisitor; +interface nsIEventTarget; + +/** + * @deprecated + * + * IMPORTANT NOTE: THIS INTERFACE IS NO LONGER SUPPORTED AND PLANNED TO BE + * REMOVED SOON. WE STRONGLY ENCORAGE TO MIGRATE THE EXISTING CODE AND FOR + * THE NEW CODE USE ONLY THE NEW HTTP CACHE API IN netwerk/cache2/. + */ +[scriptable, uuid(14dbe1e9-f3bc-45af-92f4-2c574fcd4e39)] +interface nsICacheService : nsISupports +{ + /** + * @throws NS_ERROR_NOT_IMPLEMENTED when the cache v2 is prefered to use. + * + * Create a cache session + * + * A cache session represents a client's access into the cache. The cache + * session is not "owned" by the cache service. Hence, it is possible to + * create duplicate cache sessions. Entries created by a cache session + * are invisible to other cache sessions, unless the cache sessions are + * equivalent. + * + * @param clientID - Specifies the name of the client using the cache. + * @param storagePolicy - Limits the storage policy for all entries + * accessed via the returned session. As a result, devices excluded + * by the storage policy will not be searched when opening entries + * from the returned session. + * @param streamBased - Indicates whether or not the data being cached + * can be represented as a stream. The storagePolicy must be + * consistent with the value of this field. For example, a non-stream- + * based cache entry can only have a storage policy of STORE_IN_MEMORY. + * @return new cache session. + */ + nsICacheSession createSession(in string clientID, + in nsCacheStoragePolicy storagePolicy, + in boolean streamBased); + + /** + * @throws NS_ERROR_NOT_IMPLEMENTED when the cache v2 is prefered to use. + * + * Visit entries stored in the cache. Used to implement about:cache. + */ + void visitEntries(in nsICacheVisitor visitor); + + /** + * @throws NS_ERROR_NOT_IMPLEMENTED when the cache v2 is prefered to use. + * + * Evicts all entries in all devices implied by the storage policy. + * + * @note This function may evict some items but will throw if it fails to evict + * everything. + */ + void evictEntries(in nsCacheStoragePolicy storagePolicy); + + /** + * Event target which is used for I/O operations + */ + readonly attribute nsIEventTarget cacheIOTarget; +}; + +%{C++ +/** + * Observer service notification that is sent when + * nsICacheService::evictEntries() or nsICacheSession::evictEntries() + * is called. + */ +#define NS_CACHESERVICE_EMPTYCACHE_TOPIC_ID "cacheservice:empty-cache" +%} + +[scriptable, builtinclass, uuid(d0fc8d38-db80-4928-bf1c-b0085ddfa9dc)] +interface nsICacheServiceInternal : nsICacheService +{ + /** + * This is an internal interface. It changes so frequently that it probably + * went away while you were reading this. + */ + + /** + * Milliseconds for which the service lock has been held. 0 if unlocked. + */ + readonly attribute double lockHeldTime; +}; + + diff --git a/netwerk/cache/nsICacheSession.idl b/netwerk/cache/nsICacheSession.idl new file mode 100644 index 000000000..e2a4dec5b --- /dev/null +++ b/netwerk/cache/nsICacheSession.idl @@ -0,0 +1,88 @@ +/* -*- Mode: IDL; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" +#include "nsICache.idl" + +interface nsICacheEntryDescriptor; +interface nsICacheListener; +interface nsIFile; + +[scriptable, uuid(1dd7708c-de48-4ffe-b5aa-cd218c762887)] +interface nsICacheSession : nsISupports +{ + /** + * Expired entries will be doomed or evicted if this attribute is set to + * true. If false, expired entries will be returned (useful for offline- + * mode and clients, such as HTTP, that can update the valid lifetime of + * cached content). This attribute defaults to true. + */ + attribute boolean doomEntriesIfExpired; + + /** + * When set, entries created with this session will be placed to a cache + * based at this directory. Use when storing entries to a different + * profile than the active profile of the the current running application + * process. + */ + attribute nsIFile profileDirectory; + + /** + * A cache session can only give out one descriptor with WRITE access + * to a given cache entry at a time. Until the client calls MarkValid on + * its descriptor, other attempts to open the same cache entry will block. + */ + + /** + * Synchronous cache access. This method fails if it is called on the main + * thread. Use asyncOpenCacheEntry() instead. This returns a unique + * descriptor each time it is called, even if the same key is specified. + * When called by multiple threads for write access, only one writable + * descriptor will be granted. If 'blockingMode' is set to false, it will + * return NS_ERROR_CACHE_WAIT_FOR_VALIDATION rather than block when another + * descriptor has been given WRITE access but hasn't validated the entry yet. + */ + nsICacheEntryDescriptor openCacheEntry(in ACString key, + in nsCacheAccessMode accessRequested, + in boolean blockingMode); + + /** + * Asynchronous cache access. Does not block the calling thread. Instead, + * the listener will be notified when the descriptor is available. If + * 'noWait' is set to true, the listener will be notified immediately with + * status NS_ERROR_CACHE_WAIT_FOR_VALIDATION rather than queuing the request + * when another descriptor has been given WRITE access but hasn't validated + * the entry yet. + */ + void asyncOpenCacheEntry(in ACString key, + in nsCacheAccessMode accessRequested, + in nsICacheListener listener, + [optional] in boolean noWait); + + /** + * Evict all entries for this session's clientID according to its storagePolicy. + */ + void evictEntries(); + + /** + * Return whether any of the cache devices implied by the session storage policy + * are currently enabled for instantiation if they don't already exist. + */ + boolean isStorageEnabled(); + + /** + * Asynchronously doom an entry specified by the key. Listener will be + * notified about the status of the operation. Null may be passed if caller + * doesn't care about the result. + */ + void doomEntry(in ACString key, in nsICacheListener listener); + + /** + * Private entries will be doomed when the last private browsing session + * finishes. + */ + attribute boolean isPrivate; +}; diff --git a/netwerk/cache/nsICacheVisitor.idl b/netwerk/cache/nsICacheVisitor.idl new file mode 100644 index 000000000..ddc993635 --- /dev/null +++ b/netwerk/cache/nsICacheVisitor.idl @@ -0,0 +1,123 @@ +/* -*- Mode: IDL; tab-width: 4; indent-tabs-mode: nil; c-basic-offset: 4 -*- + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsISupports.idl" + +/* XXX we should define device and entry info as well (stats, etc) */ + +interface nsICacheDeviceInfo; +interface nsICacheEntryInfo; + + +[scriptable, uuid(f8c08c4b-d778-49d1-a59b-866fdc500d95)] +interface nsICacheVisitor : nsISupports +{ + /** + * Called to provide information about a cache device. + * + * @param deviceID - specifies the device being visited. + * @param deviceInfo - specifies information about this device. + * + * @return true to start visiting all entries for this device. + * @return false to advance to the next device. + */ + boolean visitDevice(in string deviceID, + in nsICacheDeviceInfo deviceInfo); + + /** + * Called to provide information about a cache entry. + * + * @param deviceID - specifies the device being visited. + * @param entryInfo - specifies information about this entry. + * + * @return true to visit the next entry on the current device, or if the + * end of the device has been reached, advance to the next device. + * @return false to advance to the next device. + */ + boolean visitEntry(in string deviceID, + in nsICacheEntryInfo entryInfo); +}; + + +[scriptable, uuid(31d1c294-1dd2-11b2-be3a-c79230dca297)] +interface nsICacheDeviceInfo : nsISupports +{ + /** + * Get a human readable description of the cache device. + */ + readonly attribute string description; + + /** + * Get a usage report, statistics, miscellaneous data about + * the cache device. + */ + readonly attribute string usageReport; + + /** + * Get the number of stored cache entries. + */ + readonly attribute unsigned long entryCount; + + /** + * Get the total size of the stored cache entries. + */ + readonly attribute unsigned long totalSize; + + /** + * Get the upper limit of the size of the data the cache can store. + */ + readonly attribute unsigned long maximumSize; +}; + + +[scriptable, uuid(fab51c92-95c3-4468-b317-7de4d7588254)] +interface nsICacheEntryInfo : nsISupports +{ + /** + * Get the client id associated with this cache entry. + */ + readonly attribute string clientID; + + /** + * Get the id for the device that stores this cache entry. + */ + readonly attribute string deviceID; + + /** + * Get the key identifying the cache entry. + */ + readonly attribute ACString key; + + /** + * Get the number of times the cache entry has been opened. + */ + readonly attribute long fetchCount; + + /** + * Get the last time the cache entry was opened (in seconds since the Epoch). + */ + readonly attribute uint32_t lastFetched; + + /** + * Get the last time the cache entry was modified (in seconds since the Epoch). + */ + readonly attribute uint32_t lastModified; + + /** + * Get the expiration time of the cache entry (in seconds since the Epoch). + */ + readonly attribute uint32_t expirationTime; + + /** + * Get the cache entry data size. + */ + readonly attribute unsigned long dataSize; + + /** + * Find out whether or not the cache entry is stream based. + */ + boolean isStreamBased(); +}; diff --git a/netwerk/cache/nsMemoryCacheDevice.cpp b/netwerk/cache/nsMemoryCacheDevice.cpp new file mode 100644 index 000000000..042e86022 --- /dev/null +++ b/netwerk/cache/nsMemoryCacheDevice.cpp @@ -0,0 +1,617 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* vim: set ts=8 sts=4 et sw=4 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsCache.h" +#include "nsMemoryCacheDevice.h" +#include "nsCacheService.h" +#include "nsICacheService.h" +#include "nsICacheVisitor.h" +#include "nsIStorageStream.h" +#include "nsCRT.h" +#include "nsReadableUtils.h" +#include "mozilla/MathAlgorithms.h" +#include "mozilla/Telemetry.h" +#include <algorithm> + +// The memory cache implements the "LRU-SP" caching algorithm +// described in "LRU-SP: A Size-Adjusted and Popularity-Aware LRU Replacement +// Algorithm for Web Caching" by Kai Cheng and Yahiko Kambayashi. + +// We keep kQueueCount LRU queues, which should be about ceil(log2(mHardLimit)) +// The queues hold exponentially increasing ranges of floor(log2((size/nref))) +// values for entries. +// Entries larger than 2^(kQueueCount-1) go in the last queue. +// Entries with no expiration go in the first queue. + +const char *gMemoryDeviceID = "memory"; +using namespace mozilla; + +nsMemoryCacheDevice::nsMemoryCacheDevice() + : mInitialized(false), + mHardLimit(4 * 1024 * 1024), // default, if no pref + mSoftLimit((mHardLimit * 9) / 10), // default, if no pref + mTotalSize(0), + mInactiveSize(0), + mEntryCount(0), + mMaxEntryCount(0), + mMaxEntrySize(-1) // -1 means "no limit" +{ + for (int i=0; i<kQueueCount; ++i) + PR_INIT_CLIST(&mEvictionList[i]); +} + + +nsMemoryCacheDevice::~nsMemoryCacheDevice() +{ + Shutdown(); +} + + +nsresult +nsMemoryCacheDevice::Init() +{ + if (mInitialized) return NS_ERROR_ALREADY_INITIALIZED; + + mMemCacheEntries.Init(); + mInitialized = true; + return NS_OK; +} + + +nsresult +nsMemoryCacheDevice::Shutdown() +{ + NS_ASSERTION(mInitialized, "### attempting shutdown while not initialized"); + NS_ENSURE_TRUE(mInitialized, NS_ERROR_NOT_INITIALIZED); + + mMemCacheEntries.Shutdown(); + + // evict all entries + nsCacheEntry * entry, * next; + + for (int i = kQueueCount - 1; i >= 0; --i) { + entry = (nsCacheEntry *)PR_LIST_HEAD(&mEvictionList[i]); + while (entry != &mEvictionList[i]) { + NS_ASSERTION(!entry->IsInUse(), "### shutting down with active entries"); + next = (nsCacheEntry *)PR_NEXT_LINK(entry); + PR_REMOVE_AND_INIT_LINK(entry); + + // update statistics + int32_t memoryRecovered = (int32_t)entry->DataSize(); + mTotalSize -= memoryRecovered; + mInactiveSize -= memoryRecovered; + --mEntryCount; + + delete entry; + entry = next; + } + } + +/* + * we're not factoring in changes to meta data yet... + * NS_ASSERTION(mTotalSize == 0, "### mem cache leaking entries?"); + */ + NS_ASSERTION(mInactiveSize == 0, "### mem cache leaking entries?"); + NS_ASSERTION(mEntryCount == 0, "### mem cache leaking entries?"); + + mInitialized = false; + + return NS_OK; +} + + +const char * +nsMemoryCacheDevice::GetDeviceID() +{ + return gMemoryDeviceID; +} + + +nsCacheEntry * +nsMemoryCacheDevice::FindEntry(nsCString * key, bool *collision) +{ + mozilla::Telemetry::AutoTimer<mozilla::Telemetry::CACHE_MEMORY_SEARCH_2> timer; + nsCacheEntry * entry = mMemCacheEntries.GetEntry(key); + if (!entry) return nullptr; + + // move entry to the tail of an eviction list + PR_REMOVE_AND_INIT_LINK(entry); + PR_APPEND_LINK(entry, &mEvictionList[EvictionList(entry, 0)]); + + mInactiveSize -= entry->DataSize(); + + return entry; +} + + +nsresult +nsMemoryCacheDevice::DeactivateEntry(nsCacheEntry * entry) +{ + CACHE_LOG_DEBUG(("nsMemoryCacheDevice::DeactivateEntry for entry 0x%p\n", + entry)); + if (entry->IsDoomed()) { +#ifdef DEBUG + // XXX verify we've removed it from mMemCacheEntries & eviction list +#endif + delete entry; + CACHE_LOG_DEBUG(("deleted doomed entry 0x%p\n", entry)); + return NS_OK; + } + +#ifdef DEBUG + nsCacheEntry * ourEntry = mMemCacheEntries.GetEntry(entry->Key()); + NS_ASSERTION(ourEntry, "DeactivateEntry called for an entry we don't have!"); + NS_ASSERTION(entry == ourEntry, "entry doesn't match ourEntry"); + if (ourEntry != entry) + return NS_ERROR_INVALID_POINTER; +#endif + + mInactiveSize += entry->DataSize(); + EvictEntriesIfNecessary(); + + return NS_OK; +} + + +nsresult +nsMemoryCacheDevice::BindEntry(nsCacheEntry * entry) +{ + if (!entry->IsDoomed()) { + NS_ASSERTION(PR_CLIST_IS_EMPTY(entry),"entry is already on a list!"); + + // append entry to the eviction list + PR_APPEND_LINK(entry, &mEvictionList[EvictionList(entry, 0)]); + + // add entry to hashtable of mem cache entries + nsresult rv = mMemCacheEntries.AddEntry(entry); + if (NS_FAILED(rv)) { + PR_REMOVE_AND_INIT_LINK(entry); + return rv; + } + + // add size of entry to memory totals + ++mEntryCount; + if (mMaxEntryCount < mEntryCount) mMaxEntryCount = mEntryCount; + + mTotalSize += entry->DataSize(); + EvictEntriesIfNecessary(); + } + + return NS_OK; +} + + +void +nsMemoryCacheDevice::DoomEntry(nsCacheEntry * entry) +{ +#ifdef DEBUG + // debug code to verify we have entry + nsCacheEntry * hashEntry = mMemCacheEntries.GetEntry(entry->Key()); + if (!hashEntry) NS_WARNING("no entry for key"); + else if (entry != hashEntry) NS_WARNING("entry != hashEntry"); +#endif + CACHE_LOG_DEBUG(("Dooming entry 0x%p in memory cache\n", entry)); + EvictEntry(entry, DO_NOT_DELETE_ENTRY); +} + + +nsresult +nsMemoryCacheDevice::OpenInputStreamForEntry( nsCacheEntry * entry, + nsCacheAccessMode mode, + uint32_t offset, + nsIInputStream ** result) +{ + NS_ENSURE_ARG_POINTER(entry); + NS_ENSURE_ARG_POINTER(result); + + nsCOMPtr<nsIStorageStream> storage; + nsresult rv; + + nsISupports *data = entry->Data(); + if (data) { + storage = do_QueryInterface(data, &rv); + if (NS_FAILED(rv)) + return rv; + } + else { + rv = NS_NewStorageStream(4096, uint32_t(-1), getter_AddRefs(storage)); + if (NS_FAILED(rv)) + return rv; + entry->SetData(storage); + } + + return storage->NewInputStream(offset, result); +} + + +nsresult +nsMemoryCacheDevice::OpenOutputStreamForEntry( nsCacheEntry * entry, + nsCacheAccessMode mode, + uint32_t offset, + nsIOutputStream ** result) +{ + NS_ENSURE_ARG_POINTER(entry); + NS_ENSURE_ARG_POINTER(result); + + nsCOMPtr<nsIStorageStream> storage; + nsresult rv; + + nsISupports *data = entry->Data(); + if (data) { + storage = do_QueryInterface(data, &rv); + if (NS_FAILED(rv)) + return rv; + } + else { + rv = NS_NewStorageStream(4096, uint32_t(-1), getter_AddRefs(storage)); + if (NS_FAILED(rv)) + return rv; + entry->SetData(storage); + } + + return storage->GetOutputStream(offset, result); +} + + +nsresult +nsMemoryCacheDevice::GetFileForEntry( nsCacheEntry * entry, + nsIFile ** result ) +{ + return NS_ERROR_NOT_IMPLEMENTED; +} + +bool +nsMemoryCacheDevice::EntryIsTooBig(int64_t entrySize) +{ + CACHE_LOG_DEBUG(("nsMemoryCacheDevice::EntryIsTooBig " + "[size=%d max=%d soft=%d]\n", + entrySize, mMaxEntrySize, mSoftLimit)); + if (mMaxEntrySize == -1) + return entrySize > mSoftLimit; + else + return (entrySize > mSoftLimit || entrySize > mMaxEntrySize); +} + +size_t +nsMemoryCacheDevice::TotalSize() +{ + return mTotalSize; +} + +nsresult +nsMemoryCacheDevice::OnDataSizeChange( nsCacheEntry * entry, int32_t deltaSize) +{ + if (entry->IsStreamData()) { + // we have the right to refuse or pre-evict + uint32_t newSize = entry->DataSize() + deltaSize; + if (EntryIsTooBig(newSize)) { +#ifdef DEBUG + nsresult rv = +#endif + nsCacheService::DoomEntry(entry); + NS_ASSERTION(NS_SUCCEEDED(rv),"DoomEntry() failed."); + return NS_ERROR_ABORT; + } + } + + // adjust our totals + mTotalSize += deltaSize; + + if (!entry->IsDoomed()) { + // move entry to the tail of the appropriate eviction list + PR_REMOVE_AND_INIT_LINK(entry); + PR_APPEND_LINK(entry, &mEvictionList[EvictionList(entry, deltaSize)]); + } + + EvictEntriesIfNecessary(); + return NS_OK; +} + + +void +nsMemoryCacheDevice::AdjustMemoryLimits(int32_t softLimit, int32_t hardLimit) +{ + mSoftLimit = softLimit; + mHardLimit = hardLimit; + + // First, evict entries that won't fit into the new cache size. + EvictEntriesIfNecessary(); +} + + +void +nsMemoryCacheDevice::EvictEntry(nsCacheEntry * entry, bool deleteEntry) +{ + CACHE_LOG_DEBUG(("Evicting entry 0x%p from memory cache, deleting: %d\n", + entry, deleteEntry)); + // remove entry from our hashtable + mMemCacheEntries.RemoveEntry(entry); + + // remove entry from the eviction list + PR_REMOVE_AND_INIT_LINK(entry); + + // update statistics + int32_t memoryRecovered = (int32_t)entry->DataSize(); + mTotalSize -= memoryRecovered; + if (!entry->IsDoomed()) + mInactiveSize -= memoryRecovered; + --mEntryCount; + + if (deleteEntry) delete entry; +} + + +void +nsMemoryCacheDevice::EvictEntriesIfNecessary(void) +{ + nsCacheEntry * entry; + nsCacheEntry * maxEntry; + CACHE_LOG_DEBUG(("EvictEntriesIfNecessary. mTotalSize: %d, mHardLimit: %d," + "mInactiveSize: %d, mSoftLimit: %d\n", + mTotalSize, mHardLimit, mInactiveSize, mSoftLimit)); + + if ((mTotalSize < mHardLimit) && (mInactiveSize < mSoftLimit)) + return; + + uint32_t now = SecondsFromPRTime(PR_Now()); + uint64_t entryCost = 0; + uint64_t maxCost = 0; + do { + // LRU-SP eviction selection: Check the head of each segment (each + // eviction list, kept in LRU order) and select the maximal-cost + // entry for eviction. Cost is time-since-accessed * size / nref. + maxEntry = 0; + for (int i = kQueueCount - 1; i >= 0; --i) { + entry = (nsCacheEntry *)PR_LIST_HEAD(&mEvictionList[i]); + + // If the head of a list is in use, check the next available entry + while ((entry != &mEvictionList[i]) && + (entry->IsInUse())) { + entry = (nsCacheEntry *)PR_NEXT_LINK(entry); + } + + if (entry != &mEvictionList[i]) { + entryCost = (uint64_t) + (now - entry->LastFetched()) * entry->DataSize() / + std::max(1, entry->FetchCount()); + if (!maxEntry || (entryCost > maxCost)) { + maxEntry = entry; + maxCost = entryCost; + } + } + } + if (maxEntry) { + EvictEntry(maxEntry, DELETE_ENTRY); + } else { + break; + } + } + while ((mTotalSize >= mHardLimit) || (mInactiveSize >= mSoftLimit)); +} + + +int +nsMemoryCacheDevice::EvictionList(nsCacheEntry * entry, int32_t deltaSize) +{ + // favor items which never expire by putting them in the lowest-index queue + if (entry->ExpirationTime() == nsICache::NO_EXPIRATION_TIME) + return 0; + + // compute which eviction queue this entry should go into, + // based on floor(log2(size/nref)) + int32_t size = deltaSize + (int32_t)entry->DataSize(); + int32_t fetchCount = std::max(1, entry->FetchCount()); + + return std::min((int)mozilla::FloorLog2(size / fetchCount), kQueueCount - 1); +} + + +nsresult +nsMemoryCacheDevice::Visit(nsICacheVisitor * visitor) +{ + nsMemoryCacheDeviceInfo * deviceInfo = new nsMemoryCacheDeviceInfo(this); + nsCOMPtr<nsICacheDeviceInfo> deviceRef(deviceInfo); + if (!deviceInfo) return NS_ERROR_OUT_OF_MEMORY; + + bool keepGoing; + nsresult rv = visitor->VisitDevice(gMemoryDeviceID, deviceInfo, &keepGoing); + if (NS_FAILED(rv)) return rv; + + if (!keepGoing) + return NS_OK; + + nsCacheEntry * entry; + nsCOMPtr<nsICacheEntryInfo> entryRef; + + for (int i = kQueueCount - 1; i >= 0; --i) { + entry = (nsCacheEntry *)PR_LIST_HEAD(&mEvictionList[i]); + while (entry != &mEvictionList[i]) { + nsCacheEntryInfo * entryInfo = new nsCacheEntryInfo(entry); + if (!entryInfo) return NS_ERROR_OUT_OF_MEMORY; + entryRef = entryInfo; + + rv = visitor->VisitEntry(gMemoryDeviceID, entryInfo, &keepGoing); + entryInfo->DetachEntry(); + if (NS_FAILED(rv)) return rv; + if (!keepGoing) break; + + entry = (nsCacheEntry *)PR_NEXT_LINK(entry); + } + } + return NS_OK; +} + + +static bool +IsEntryPrivate(nsCacheEntry* entry, void* args) +{ + return entry->IsPrivate(); +} + +struct ClientIDArgs { + const char* clientID; + uint32_t prefixLength; +}; + +static bool +EntryMatchesClientID(nsCacheEntry* entry, void* args) +{ + const char * clientID = static_cast<ClientIDArgs*>(args)->clientID; + uint32_t prefixLength = static_cast<ClientIDArgs*>(args)->prefixLength; + const char * key = entry->Key()->get(); + return !clientID || nsCRT::strncmp(clientID, key, prefixLength) == 0; +} + +nsresult +nsMemoryCacheDevice::DoEvictEntries(bool (*matchFn)(nsCacheEntry* entry, void* args), void* args) +{ + nsCacheEntry * entry; + + for (int i = kQueueCount - 1; i >= 0; --i) { + PRCList * elem = PR_LIST_HEAD(&mEvictionList[i]); + while (elem != &mEvictionList[i]) { + entry = (nsCacheEntry *)elem; + elem = PR_NEXT_LINK(elem); + + if (!matchFn(entry, args)) + continue; + + if (entry->IsInUse()) { + nsresult rv = nsCacheService::DoomEntry(entry); + if (NS_FAILED(rv)) { + CACHE_LOG_WARNING(("memCache->DoEvictEntries() aborted: rv =%x", rv)); + return rv; + } + } else { + EvictEntry(entry, DELETE_ENTRY); + } + } + } + + return NS_OK; +} + +nsresult +nsMemoryCacheDevice::EvictEntries(const char * clientID) +{ + ClientIDArgs args = {clientID, clientID ? uint32_t(strlen(clientID)) : 0}; + return DoEvictEntries(&EntryMatchesClientID, &args); +} + +nsresult +nsMemoryCacheDevice::EvictPrivateEntries() +{ + return DoEvictEntries(&IsEntryPrivate, nullptr); +} + + +// WARNING: SetCapacity can get called before Init() +void +nsMemoryCacheDevice::SetCapacity(int32_t capacity) +{ + int32_t hardLimit = capacity * 1024; // convert k into bytes + int32_t softLimit = (hardLimit * 9) / 10; + AdjustMemoryLimits(softLimit, hardLimit); +} + +void +nsMemoryCacheDevice::SetMaxEntrySize(int32_t maxSizeInKilobytes) +{ + // Internal unit is bytes. Changing this only takes effect *after* the + // change and has no consequences for existing cache-entries + if (maxSizeInKilobytes >= 0) + mMaxEntrySize = maxSizeInKilobytes * 1024; + else + mMaxEntrySize = -1; +} + +#ifdef DEBUG +void +nsMemoryCacheDevice::CheckEntryCount() +{ + if (!mInitialized) return; + + int32_t evictionListCount = 0; + for (int i=0; i<kQueueCount; ++i) { + PRCList * elem = PR_LIST_HEAD(&mEvictionList[i]); + while (elem != &mEvictionList[i]) { + elem = PR_NEXT_LINK(elem); + ++evictionListCount; + } + } + NS_ASSERTION(mEntryCount == evictionListCount, "### mem cache badness"); + + int32_t entryCount = 0; + for (auto iter = mMemCacheEntries.Iter(); !iter.Done(); iter.Next()) { + ++entryCount; + } + NS_ASSERTION(mEntryCount == entryCount, "### mem cache badness"); +} +#endif + +/****************************************************************************** + * nsMemoryCacheDeviceInfo - for implementing about:cache + *****************************************************************************/ + + +NS_IMPL_ISUPPORTS(nsMemoryCacheDeviceInfo, nsICacheDeviceInfo) + + +NS_IMETHODIMP +nsMemoryCacheDeviceInfo::GetDescription(char ** result) +{ + NS_ENSURE_ARG_POINTER(result); + *result = NS_strdup("Memory cache device"); + if (!*result) return NS_ERROR_OUT_OF_MEMORY; + return NS_OK; +} + + +NS_IMETHODIMP +nsMemoryCacheDeviceInfo::GetUsageReport(char ** result) +{ + NS_ENSURE_ARG_POINTER(result); + nsCString buffer; + + buffer.AssignLiteral(" <tr>\n" + " <th>Inactive storage:</th>\n" + " <td>"); + buffer.AppendInt(mDevice->mInactiveSize / 1024); + buffer.AppendLiteral(" KiB</td>\n" + " </tr>\n"); + + *result = ToNewCString(buffer); + if (!*result) return NS_ERROR_OUT_OF_MEMORY; + return NS_OK; +} + + +NS_IMETHODIMP +nsMemoryCacheDeviceInfo::GetEntryCount(uint32_t * result) +{ + NS_ENSURE_ARG_POINTER(result); + // XXX compare calculated count vs. mEntryCount + *result = (uint32_t)mDevice->mEntryCount; + return NS_OK; +} + + +NS_IMETHODIMP +nsMemoryCacheDeviceInfo::GetTotalSize(uint32_t * result) +{ + NS_ENSURE_ARG_POINTER(result); + *result = (uint32_t)mDevice->mTotalSize; + return NS_OK; +} + + +NS_IMETHODIMP +nsMemoryCacheDeviceInfo::GetMaximumSize(uint32_t * result) +{ + NS_ENSURE_ARG_POINTER(result); + *result = (uint32_t)mDevice->mHardLimit; + return NS_OK; +} diff --git a/netwerk/cache/nsMemoryCacheDevice.h b/netwerk/cache/nsMemoryCacheDevice.h new file mode 100644 index 000000000..331b04ba6 --- /dev/null +++ b/netwerk/cache/nsMemoryCacheDevice.h @@ -0,0 +1,124 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 4 -*- */ +/* vim: set ts=8 sts=4 et sw=4 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef _nsMemoryCacheDevice_h_ +#define _nsMemoryCacheDevice_h_ + +#include "nsCacheDevice.h" +#include "PLDHashTable.h" +#include "nsCacheEntry.h" + + +class nsMemoryCacheDeviceInfo; + +/****************************************************************************** + * nsMemoryCacheDevice + ******************************************************************************/ +class nsMemoryCacheDevice : public nsCacheDevice +{ +public: + nsMemoryCacheDevice(); + virtual ~nsMemoryCacheDevice(); + + virtual nsresult Init(); + virtual nsresult Shutdown(); + + virtual const char * GetDeviceID(void); + + virtual nsresult BindEntry( nsCacheEntry * entry ); + virtual nsCacheEntry * FindEntry( nsCString * key, bool *collision ); + virtual void DoomEntry( nsCacheEntry * entry ); + virtual nsresult DeactivateEntry( nsCacheEntry * entry ); + + virtual nsresult OpenInputStreamForEntry(nsCacheEntry * entry, + nsCacheAccessMode mode, + uint32_t offset, + nsIInputStream ** result); + + virtual nsresult OpenOutputStreamForEntry(nsCacheEntry * entry, + nsCacheAccessMode mode, + uint32_t offset, + nsIOutputStream ** result); + + virtual nsresult GetFileForEntry( nsCacheEntry * entry, + nsIFile ** result ); + + virtual nsresult OnDataSizeChange( nsCacheEntry * entry, int32_t deltaSize ); + + virtual nsresult Visit( nsICacheVisitor * visitor ); + + virtual nsresult EvictEntries(const char * clientID); + nsresult EvictPrivateEntries(); + + void SetCapacity(int32_t capacity); + void SetMaxEntrySize(int32_t maxSizeInKilobytes); + + bool EntryIsTooBig(int64_t entrySize); + + size_t TotalSize(); + +private: + friend class nsMemoryCacheDeviceInfo; + enum { DELETE_ENTRY = true, + DO_NOT_DELETE_ENTRY = false }; + + void AdjustMemoryLimits( int32_t softLimit, int32_t hardLimit); + void EvictEntry( nsCacheEntry * entry , bool deleteEntry); + void EvictEntriesIfNecessary(); + int EvictionList(nsCacheEntry * entry, int32_t deltaSize); + + typedef bool (*EvictionMatcherFn)(nsCacheEntry* entry, void* args); + nsresult DoEvictEntries(EvictionMatcherFn matchFn, void* args); + +#ifdef DEBUG + void CheckEntryCount(); +#endif + /* + * Data members + */ + enum { + kQueueCount = 24 // entries > 2^23 (8Mb) start in last queue + }; + + nsCacheEntryHashTable mMemCacheEntries; + bool mInitialized; + + PRCList mEvictionList[kQueueCount]; + + int32_t mHardLimit; + int32_t mSoftLimit; + + int32_t mTotalSize; + int32_t mInactiveSize; + + int32_t mEntryCount; + int32_t mMaxEntryCount; + int32_t mMaxEntrySize; // internal unit is bytes + + // XXX what other stats do we want to keep? +}; + + +/****************************************************************************** + * nsMemoryCacheDeviceInfo - used to call nsIVisitor for about:cache + ******************************************************************************/ +class nsMemoryCacheDeviceInfo : public nsICacheDeviceInfo { +public: + NS_DECL_ISUPPORTS + NS_DECL_NSICACHEDEVICEINFO + + explicit nsMemoryCacheDeviceInfo(nsMemoryCacheDevice* device) + : mDevice(device) + { + } + +private: + virtual ~nsMemoryCacheDeviceInfo() {} + nsMemoryCacheDevice* mDevice; +}; + + +#endif // _nsMemoryCacheDevice_h_ |