From 5f8de423f190bbb79a62f804151bc24824fa32d8 Mon Sep 17 00:00:00 2001 From: "Matt A. Tobin" Date: Fri, 2 Feb 2018 04:16:08 -0500 Subject: Add m-esr52 at 52.6.0 --- startupcache/StartupCache.cpp | 835 +++++++++++++++++++++ startupcache/StartupCache.h | 230 ++++++ startupcache/StartupCacheModule.cpp | 45 ++ startupcache/StartupCacheUtils.cpp | 253 +++++++ startupcache/StartupCacheUtils.h | 43 ++ startupcache/moz.build | 28 + startupcache/nsIStartupCache.idl | 65 ++ startupcache/test/TestStartupCache.cpp | 480 ++++++++++++ startupcache/test/TestStartupCacheTelemetry.js | 60 ++ .../test/TestStartupCacheTelemetry.manifest | 2 + startupcache/test/moz.build | 14 + 11 files changed, 2055 insertions(+) create mode 100644 startupcache/StartupCache.cpp create mode 100644 startupcache/StartupCache.h create mode 100644 startupcache/StartupCacheModule.cpp create mode 100644 startupcache/StartupCacheUtils.cpp create mode 100644 startupcache/StartupCacheUtils.h create mode 100644 startupcache/moz.build create mode 100644 startupcache/nsIStartupCache.idl create mode 100644 startupcache/test/TestStartupCache.cpp create mode 100644 startupcache/test/TestStartupCacheTelemetry.js create mode 100644 startupcache/test/TestStartupCacheTelemetry.manifest create mode 100644 startupcache/test/moz.build (limited to 'startupcache') diff --git a/startupcache/StartupCache.cpp b/startupcache/StartupCache.cpp new file mode 100644 index 000000000..371f4795c --- /dev/null +++ b/startupcache/StartupCache.cpp @@ -0,0 +1,835 @@ +/* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* vim: set ts=8 sts=2 et sw=2 tw=80: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "prio.h" +#include "PLDHashTable.h" +#include "nsXPCOMStrings.h" +#include "mozilla/IOInterposer.h" +#include "mozilla/MemoryReporting.h" +#include "mozilla/scache/StartupCache.h" + +#include "nsAutoPtr.h" +#include "nsClassHashtable.h" +#include "nsComponentManagerUtils.h" +#include "nsDirectoryServiceUtils.h" +#include "nsIClassInfo.h" +#include "nsIFile.h" +#include "nsIObserver.h" +#include "nsIObserverService.h" +#include "nsIOutputStream.h" +#include "nsIStartupCache.h" +#include "nsIStorageStream.h" +#include "nsIStreamBufferAccess.h" +#include "nsIStringStream.h" +#include "nsISupports.h" +#include "nsITimer.h" +#include "nsIZipWriter.h" +#include "nsIZipReader.h" +#include "nsWeakReference.h" +#include "nsZipArchive.h" +#include "mozilla/Omnijar.h" +#include "prenv.h" +#include "mozilla/Telemetry.h" +#include "nsThreadUtils.h" +#include "nsXULAppAPI.h" +#include "nsIProtocolHandler.h" + +#ifdef IS_BIG_ENDIAN +#define SC_ENDIAN "big" +#else +#define SC_ENDIAN "little" +#endif + +#if PR_BYTES_PER_WORD == 4 +#define SC_WORDSIZE "4" +#else +#define SC_WORDSIZE "8" +#endif + +namespace mozilla { +namespace scache { + +MOZ_DEFINE_MALLOC_SIZE_OF(StartupCacheMallocSizeOf) + +NS_IMETHODIMP +StartupCache::CollectReports(nsIHandleReportCallback* aHandleReport, + nsISupports* aData, bool aAnonymize) +{ + MOZ_COLLECT_REPORT( + "explicit/startup-cache/mapping", KIND_NONHEAP, UNITS_BYTES, + SizeOfMapping(), + "Memory used to hold the mapping of the startup cache from file. " + "This memory is likely to be swapped out shortly after start-up."); + + MOZ_COLLECT_REPORT( + "explicit/startup-cache/data", KIND_HEAP, UNITS_BYTES, + HeapSizeOfIncludingThis(StartupCacheMallocSizeOf), + "Memory used by the startup cache for things other than the file mapping."); + + return NS_OK; +} + +#define STARTUP_CACHE_NAME "startupCache." SC_WORDSIZE "." SC_ENDIAN + +StartupCache* +StartupCache::GetSingleton() +{ + if (!gStartupCache) { + if (!XRE_IsParentProcess()) { + return nullptr; + } +#ifdef MOZ_DISABLE_STARTUPCACHE + return nullptr; +#else + StartupCache::InitSingleton(); +#endif + } + + return StartupCache::gStartupCache; +} + +void +StartupCache::DeleteSingleton() +{ + StartupCache::gStartupCache = nullptr; +} + +nsresult +StartupCache::InitSingleton() +{ + nsresult rv; + StartupCache::gStartupCache = new StartupCache(); + + rv = StartupCache::gStartupCache->Init(); + if (NS_FAILED(rv)) { + StartupCache::gStartupCache = nullptr; + } + return rv; +} + +StaticRefPtr StartupCache::gStartupCache; +bool StartupCache::gShutdownInitiated; +bool StartupCache::gIgnoreDiskCache; +enum StartupCache::TelemetrifyAge StartupCache::gPostFlushAgeAction = StartupCache::IGNORE_AGE; + +NS_IMPL_ISUPPORTS(StartupCache, nsIMemoryReporter) + +StartupCache::StartupCache() + : mArchive(nullptr), mStartupWriteInitiated(false), mWriteThread(nullptr) +{ } + +StartupCache::~StartupCache() +{ + if (mTimer) { + mTimer->Cancel(); + } + + // Generally, the in-memory table should be empty here, + // but an early shutdown means either mTimer didn't run + // or the write thread is still running. + WaitOnWriteThread(); + + // If we shutdown quickly timer wont have fired. Instead of writing + // it on the main thread and block the shutdown we simply wont update + // the startup cache. Always do this if the file doesn't exist since + // we use it part of the package step. + if (!mArchive) { + WriteToDisk(); + } + + UnregisterWeakMemoryReporter(this); +} + +nsresult +StartupCache::Init() +{ + // workaround for bug 653936 + nsCOMPtr jarInitializer(do_GetService(NS_NETWORK_PROTOCOL_CONTRACTID_PREFIX "jar")); + + nsresult rv; + + // This allows to override the startup cache filename + // which is useful from xpcshell, when there is no ProfLDS directory to keep cache in. + char *env = PR_GetEnv("MOZ_STARTUP_CACHE"); + if (env) { + rv = NS_NewLocalFile(NS_ConvertUTF8toUTF16(env), false, getter_AddRefs(mFile)); + } else { + nsCOMPtr file; + rv = NS_GetSpecialDirectory("ProfLDS", + getter_AddRefs(file)); + if (NS_FAILED(rv)) { + // return silently, this will fail in mochitests's xpcshell process. + return rv; + } + + nsCOMPtr profDir; + NS_GetSpecialDirectory("ProfDS", getter_AddRefs(profDir)); + if (profDir) { + bool same; + if (NS_SUCCEEDED(profDir->Equals(file, &same)) && !same) { + // We no longer store the startup cache in the main profile + // directory, so we should cleanup the old one. + if (NS_SUCCEEDED( + profDir->AppendNative(NS_LITERAL_CSTRING("startupCache")))) { + profDir->Remove(true); + } + } + } + + rv = file->AppendNative(NS_LITERAL_CSTRING("startupCache")); + NS_ENSURE_SUCCESS(rv, rv); + + // Try to create the directory if it's not there yet + rv = file->Create(nsIFile::DIRECTORY_TYPE, 0777); + if (NS_FAILED(rv) && rv != NS_ERROR_FILE_ALREADY_EXISTS) + return rv; + + rv = file->AppendNative(NS_LITERAL_CSTRING(STARTUP_CACHE_NAME)); + + NS_ENSURE_SUCCESS(rv, rv); + + mFile = do_QueryInterface(file); + } + + NS_ENSURE_TRUE(mFile, NS_ERROR_UNEXPECTED); + + mObserverService = do_GetService("@mozilla.org/observer-service;1"); + + if (!mObserverService) { + NS_WARNING("Could not get observerService."); + return NS_ERROR_UNEXPECTED; + } + + mListener = new StartupCacheListener(); + rv = mObserverService->AddObserver(mListener, NS_XPCOM_SHUTDOWN_OBSERVER_ID, + false); + NS_ENSURE_SUCCESS(rv, rv); + rv = mObserverService->AddObserver(mListener, "startupcache-invalidate", + false); + NS_ENSURE_SUCCESS(rv, rv); + + rv = LoadArchive(RECORD_AGE); + + // Sometimes we don't have a cache yet, that's ok. + // If it's corrupted, just remove it and start over. + if (gIgnoreDiskCache || (NS_FAILED(rv) && rv != NS_ERROR_FILE_NOT_FOUND)) { + NS_WARNING("Failed to load startupcache file correctly, removing!"); + InvalidateCache(); + } + + RegisterWeakMemoryReporter(this); + + return NS_OK; +} + +/** + * LoadArchive can be called from the main thread or while reloading cache on write thread. + */ +nsresult +StartupCache::LoadArchive(enum TelemetrifyAge flag) +{ + if (gIgnoreDiskCache) + return NS_ERROR_FAILURE; + + bool exists; + mArchive = nullptr; + nsresult rv = mFile->Exists(&exists); + if (NS_FAILED(rv) || !exists) + return NS_ERROR_FILE_NOT_FOUND; + + mArchive = new nsZipArchive(); + rv = mArchive->OpenArchive(mFile); + if (NS_FAILED(rv) || flag == IGNORE_AGE) + return rv; + + nsCString comment; + if (!mArchive->GetComment(comment)) { + return rv; + } + + const char *data; + size_t len = NS_CStringGetData(comment, &data); + PRTime creationStamp; + // We might not have a comment if the startup cache file was created + // before we started recording creation times in the comment. + if (len == sizeof(creationStamp)) { + memcpy(&creationStamp, data, len); + PRTime current = PR_Now(); + int64_t diff = current - creationStamp; + + // We can't use AccumulateTimeDelta here because we have no way of + // reifying a TimeStamp from creationStamp. + int64_t usec_per_hour = PR_USEC_PER_SEC * int64_t(3600); + int64_t hour_diff = (diff + usec_per_hour - 1) / usec_per_hour; + mozilla::Telemetry::Accumulate(Telemetry::STARTUP_CACHE_AGE_HOURS, + hour_diff); + } + + return rv; +} + +namespace { + +nsresult +GetBufferFromZipArchive(nsZipArchive *zip, bool doCRC, const char* id, + UniquePtr* outbuf, uint32_t* length) +{ + if (!zip) + return NS_ERROR_NOT_AVAILABLE; + + nsZipItemPtr zipItem(zip, id, doCRC); + if (!zipItem) + return NS_ERROR_NOT_AVAILABLE; + + *outbuf = zipItem.Forget(); + *length = zipItem.Length(); + return NS_OK; +} + +} /* anonymous namespace */ + +// NOTE: this will not find a new entry until it has been written to disk! +// Consumer should take ownership of the resulting buffer. +nsresult +StartupCache::GetBuffer(const char* id, UniquePtr* outbuf, uint32_t* length) +{ + PROFILER_LABEL_FUNC(js::ProfileEntry::Category::OTHER); + + NS_ASSERTION(NS_IsMainThread(), "Startup cache only available on main thread"); + + WaitOnWriteThread(); + if (!mStartupWriteInitiated) { + CacheEntry* entry; + nsDependentCString idStr(id); + mTable.Get(idStr, &entry); + if (entry) { + *outbuf = MakeUnique(entry->size); + memcpy(outbuf->get(), entry->data.get(), entry->size); + *length = entry->size; + return NS_OK; + } + } + + nsresult rv = GetBufferFromZipArchive(mArchive, true, id, outbuf, length); + if (NS_SUCCEEDED(rv)) + return rv; + + RefPtr omnijar = mozilla::Omnijar::GetReader(mozilla::Omnijar::APP); + // no need to checksum omnijarred entries + rv = GetBufferFromZipArchive(omnijar, false, id, outbuf, length); + if (NS_SUCCEEDED(rv)) + return rv; + + omnijar = mozilla::Omnijar::GetReader(mozilla::Omnijar::GRE); + // no need to checksum omnijarred entries + return GetBufferFromZipArchive(omnijar, false, id, outbuf, length); +} + +// Makes a copy of the buffer, client retains ownership of inbuf. +nsresult +StartupCache::PutBuffer(const char* id, const char* inbuf, uint32_t len) +{ + NS_ASSERTION(NS_IsMainThread(), "Startup cache only available on main thread"); + WaitOnWriteThread(); + if (StartupCache::gShutdownInitiated) { + return NS_ERROR_NOT_AVAILABLE; + } + + auto data = MakeUnique(len); + memcpy(data.get(), inbuf, len); + + nsCString idStr(id); + // Cache it for now, we'll write all together later. + CacheEntry* entry; + + if (mTable.Get(idStr)) { + NS_WARNING("Existing entry in StartupCache."); + // Double-caching is undesirable but not an error. + return NS_OK; + } + +#ifdef DEBUG + if (mArchive) { + nsZipItem* zipItem = mArchive->GetItem(id); + NS_ASSERTION(zipItem == nullptr, "Existing entry in disk StartupCache."); + } +#endif + + entry = new CacheEntry(Move(data), len); + mTable.Put(idStr, entry); + mPendingWrites.AppendElement(idStr); + return ResetStartupWriteTimer(); +} + +size_t +StartupCache::SizeOfMapping() +{ + return mArchive ? mArchive->SizeOfMapping() : 0; +} + +size_t +StartupCache::HeapSizeOfIncludingThis(mozilla::MallocSizeOf aMallocSizeOf) const +{ + // This function could measure more members, but they haven't been found by + // DMD to be significant. They can be added later if necessary. + + size_t n = aMallocSizeOf(this); + + n += mTable.ShallowSizeOfExcludingThis(aMallocSizeOf); + for (auto iter = mTable.ConstIter(); !iter.Done(); iter.Next()) { + n += iter.Data()->SizeOfIncludingThis(aMallocSizeOf); + } + + n += mPendingWrites.ShallowSizeOfExcludingThis(aMallocSizeOf); + + return n; +} + +struct CacheWriteHolder +{ + nsCOMPtr writer; + nsCOMPtr stream; + PRTime time; +}; + +static void +CacheCloseHelper(const nsACString& key, const CacheEntry* data, + const CacheWriteHolder* holder) +{ + MOZ_ASSERT(data); // assert key was found in mTable. + + nsresult rv; + nsIStringInputStream* stream = holder->stream; + nsIZipWriter* writer = holder->writer; + + stream->ShareData(data->data.get(), data->size); + +#ifdef DEBUG + bool hasEntry; + rv = writer->HasEntry(key, &hasEntry); + NS_ASSERTION(NS_SUCCEEDED(rv) && hasEntry == false, + "Existing entry in disk StartupCache."); +#endif + rv = writer->AddEntryStream(key, holder->time, true, stream, false); + + if (NS_FAILED(rv)) { + NS_WARNING("cache entry deleted but not written to disk."); + } +} + + +/** + * WriteToDisk writes the cache out to disk. Callers of WriteToDisk need to call WaitOnWriteThread + * to make sure there isn't a write happening on another thread + */ +void +StartupCache::WriteToDisk() +{ + nsresult rv; + mStartupWriteInitiated = true; + + if (mTable.Count() == 0) + return; + + nsCOMPtr zipW = do_CreateInstance("@mozilla.org/zipwriter;1"); + if (!zipW) + return; + + rv = zipW->Open(mFile, PR_RDWR | PR_CREATE_FILE); + if (NS_FAILED(rv)) { + NS_WARNING("could not open zipfile for write"); + return; + } + + // If we didn't have an mArchive member, that means that we failed to + // open the startup cache for reading. Therefore, we need to record + // the time of creation in a zipfile comment; this will be useful for + // Telemetry statistics. + PRTime now = PR_Now(); + if (!mArchive) { + nsCString comment; + comment.Assign((char *)&now, sizeof(now)); + zipW->SetComment(comment); + } + + nsCOMPtr stream + = do_CreateInstance("@mozilla.org/io/string-input-stream;1", &rv); + if (NS_FAILED(rv)) { + NS_WARNING("Couldn't create string input stream."); + return; + } + + CacheWriteHolder holder; + holder.stream = stream; + holder.writer = zipW; + holder.time = now; + + for (auto key = mPendingWrites.begin(); key != mPendingWrites.end(); key++) { + CacheCloseHelper(*key, mTable.Get(*key), &holder); + } + mPendingWrites.Clear(); + mTable.Clear(); + + // Close the archive so Windows doesn't choke. + mArchive = nullptr; + zipW->Close(); + + // We succesfully wrote the archive to disk; mark the disk file as trusted + gIgnoreDiskCache = false; + + // Our reader's view of the archive is outdated now, reload it. + LoadArchive(gPostFlushAgeAction); + + return; +} + +void +StartupCache::InvalidateCache() +{ + WaitOnWriteThread(); + mPendingWrites.Clear(); + mTable.Clear(); + mArchive = nullptr; + nsresult rv = mFile->Remove(false); + if (NS_FAILED(rv) && rv != NS_ERROR_FILE_TARGET_DOES_NOT_EXIST && + rv != NS_ERROR_FILE_NOT_FOUND) { + gIgnoreDiskCache = true; + mozilla::Telemetry::Accumulate(Telemetry::STARTUP_CACHE_INVALID, true); + return; + } + gIgnoreDiskCache = false; + LoadArchive(gPostFlushAgeAction); +} + +void +StartupCache::IgnoreDiskCache() +{ + gIgnoreDiskCache = true; + if (gStartupCache) + gStartupCache->InvalidateCache(); +} + +/* + * WaitOnWriteThread() is called from a main thread to wait for the worker + * thread to finish. However since the same code is used in the worker thread and + * main thread, the worker thread can also call WaitOnWriteThread() which is a no-op. + */ +void +StartupCache::WaitOnWriteThread() +{ + NS_ASSERTION(NS_IsMainThread(), "Startup cache should only wait for io thread on main thread"); + if (!mWriteThread || mWriteThread == PR_GetCurrentThread()) + return; + + PR_JoinThread(mWriteThread); + mWriteThread = nullptr; +} + +void +StartupCache::ThreadedWrite(void *aClosure) +{ + PR_SetCurrentThreadName("StartupCache"); + mozilla::IOInterposer::RegisterCurrentThread(); + /* + * It is safe to use the pointer passed in aClosure to reference the + * StartupCache object because the thread's lifetime is tightly coupled to + * the lifetime of the StartupCache object; this thread is joined in the + * StartupCache destructor, guaranteeing that this function runs if and only + * if the StartupCache object is valid. + */ + StartupCache* startupCacheObj = static_cast(aClosure); + startupCacheObj->WriteToDisk(); + mozilla::IOInterposer::UnregisterCurrentThread(); +} + +/* + * The write-thread is spawned on a timeout(which is reset with every write). This + * can avoid a slow shutdown. After writing out the cache, the zipreader is + * reloaded on the worker thread. + */ +void +StartupCache::WriteTimeout(nsITimer *aTimer, void *aClosure) +{ + /* + * It is safe to use the pointer passed in aClosure to reference the + * StartupCache object because the timer's lifetime is tightly coupled to + * the lifetime of the StartupCache object; this timer is canceled in the + * StartupCache destructor, guaranteeing that this function runs if and only + * if the StartupCache object is valid. + */ + StartupCache* startupCacheObj = static_cast(aClosure); + startupCacheObj->mWriteThread = PR_CreateThread(PR_USER_THREAD, + StartupCache::ThreadedWrite, + startupCacheObj, + PR_PRIORITY_NORMAL, + PR_GLOBAL_THREAD, + PR_JOINABLE_THREAD, + 0); +} + +// We don't want to refcount StartupCache, so we'll just +// hold a ref to this and pass it to observerService instead. +NS_IMPL_ISUPPORTS(StartupCacheListener, nsIObserver) + +nsresult +StartupCacheListener::Observe(nsISupports *subject, const char* topic, const char16_t* data) +{ + StartupCache* sc = StartupCache::GetSingleton(); + if (!sc) + return NS_OK; + + if (strcmp(topic, NS_XPCOM_SHUTDOWN_OBSERVER_ID) == 0) { + // Do not leave the thread running past xpcom shutdown + sc->WaitOnWriteThread(); + StartupCache::gShutdownInitiated = true; + } else if (strcmp(topic, "startupcache-invalidate") == 0) { + sc->InvalidateCache(); + } + return NS_OK; +} + +nsresult +StartupCache::GetDebugObjectOutputStream(nsIObjectOutputStream* aStream, + nsIObjectOutputStream** aOutStream) +{ + NS_ENSURE_ARG_POINTER(aStream); +#ifdef DEBUG + StartupCacheDebugOutputStream* stream + = new StartupCacheDebugOutputStream(aStream, &mWriteObjectMap); + NS_ADDREF(*aOutStream = stream); +#else + NS_ADDREF(*aOutStream = aStream); +#endif + + return NS_OK; +} + +nsresult +StartupCache::ResetStartupWriteTimer() +{ + mStartupWriteInitiated = false; + nsresult rv; + if (!mTimer) + mTimer = do_CreateInstance("@mozilla.org/timer;1", &rv); + else + rv = mTimer->Cancel(); + NS_ENSURE_SUCCESS(rv, rv); + // Wait for 10 seconds, then write out the cache. + mTimer->InitWithFuncCallback(StartupCache::WriteTimeout, this, 60000, + nsITimer::TYPE_ONE_SHOT); + return NS_OK; +} + +nsresult +StartupCache::RecordAgesAlways() +{ + gPostFlushAgeAction = RECORD_AGE; + return NS_OK; +} + +// StartupCacheDebugOutputStream implementation +#ifdef DEBUG +NS_IMPL_ISUPPORTS(StartupCacheDebugOutputStream, nsIObjectOutputStream, + nsIBinaryOutputStream, nsIOutputStream) + +bool +StartupCacheDebugOutputStream::CheckReferences(nsISupports* aObject) +{ + nsresult rv; + + nsCOMPtr classInfo = do_QueryInterface(aObject); + if (!classInfo) { + NS_ERROR("aObject must implement nsIClassInfo"); + return false; + } + + uint32_t flags; + rv = classInfo->GetFlags(&flags); + NS_ENSURE_SUCCESS(rv, false); + if (flags & nsIClassInfo::SINGLETON) + return true; + + nsISupportsHashKey* key = mObjectMap->GetEntry(aObject); + if (key) { + NS_ERROR("non-singleton aObject is referenced multiple times in this" + "serialization, we don't support that."); + return false; + } + + mObjectMap->PutEntry(aObject); + return true; +} + +// nsIObjectOutputStream implementation +nsresult +StartupCacheDebugOutputStream::WriteObject(nsISupports* aObject, bool aIsStrongRef) +{ + nsCOMPtr rootObject(do_QueryInterface(aObject)); + + NS_ASSERTION(rootObject.get() == aObject, + "bad call to WriteObject -- call WriteCompoundObject!"); + bool check = CheckReferences(aObject); + NS_ENSURE_TRUE(check, NS_ERROR_FAILURE); + return mBinaryStream->WriteObject(aObject, aIsStrongRef); +} + +nsresult +StartupCacheDebugOutputStream::WriteSingleRefObject(nsISupports* aObject) +{ + nsCOMPtr rootObject(do_QueryInterface(aObject)); + + NS_ASSERTION(rootObject.get() == aObject, + "bad call to WriteSingleRefObject -- call WriteCompoundObject!"); + bool check = CheckReferences(aObject); + NS_ENSURE_TRUE(check, NS_ERROR_FAILURE); + return mBinaryStream->WriteSingleRefObject(aObject); +} + +nsresult +StartupCacheDebugOutputStream::WriteCompoundObject(nsISupports* aObject, + const nsIID& aIID, + bool aIsStrongRef) +{ + nsCOMPtr rootObject(do_QueryInterface(aObject)); + + nsCOMPtr roundtrip; + rootObject->QueryInterface(aIID, getter_AddRefs(roundtrip)); + NS_ASSERTION(roundtrip.get() == aObject, + "bad aggregation or multiple inheritance detected by call to " + "WriteCompoundObject!"); + + bool check = CheckReferences(aObject); + NS_ENSURE_TRUE(check, NS_ERROR_FAILURE); + return mBinaryStream->WriteCompoundObject(aObject, aIID, aIsStrongRef); +} + +nsresult +StartupCacheDebugOutputStream::WriteID(nsID const& aID) +{ + return mBinaryStream->WriteID(aID); +} + +char* +StartupCacheDebugOutputStream::GetBuffer(uint32_t aLength, uint32_t aAlignMask) +{ + return mBinaryStream->GetBuffer(aLength, aAlignMask); +} + +void +StartupCacheDebugOutputStream::PutBuffer(char* aBuffer, uint32_t aLength) +{ + mBinaryStream->PutBuffer(aBuffer, aLength); +} +#endif //DEBUG + +StartupCacheWrapper* StartupCacheWrapper::gStartupCacheWrapper = nullptr; + +NS_IMPL_ISUPPORTS(StartupCacheWrapper, nsIStartupCache) + +StartupCacheWrapper::~StartupCacheWrapper() +{ + MOZ_ASSERT(gStartupCacheWrapper == this); + gStartupCacheWrapper = nullptr; +} + +StartupCacheWrapper* StartupCacheWrapper::GetSingleton() +{ + if (!gStartupCacheWrapper) + gStartupCacheWrapper = new StartupCacheWrapper(); + + NS_ADDREF(gStartupCacheWrapper); + return gStartupCacheWrapper; +} + +nsresult +StartupCacheWrapper::GetBuffer(const char* id, char** outbuf, uint32_t* length) +{ + StartupCache* sc = StartupCache::GetSingleton(); + if (!sc) { + return NS_ERROR_NOT_INITIALIZED; + } + UniquePtr buf; + nsresult rv = sc->GetBuffer(id, &buf, length); + *outbuf = buf.release(); + return rv; +} + +nsresult +StartupCacheWrapper::PutBuffer(const char* id, const char* inbuf, uint32_t length) +{ + StartupCache* sc = StartupCache::GetSingleton(); + if (!sc) { + return NS_ERROR_NOT_INITIALIZED; + } + return sc->PutBuffer(id, inbuf, length); +} + +nsresult +StartupCacheWrapper::InvalidateCache() +{ + StartupCache* sc = StartupCache::GetSingleton(); + if (!sc) { + return NS_ERROR_NOT_INITIALIZED; + } + sc->InvalidateCache(); + return NS_OK; +} + +nsresult +StartupCacheWrapper::IgnoreDiskCache() +{ + StartupCache::IgnoreDiskCache(); + return NS_OK; +} + +nsresult +StartupCacheWrapper::GetDebugObjectOutputStream(nsIObjectOutputStream* stream, + nsIObjectOutputStream** outStream) +{ + StartupCache* sc = StartupCache::GetSingleton(); + if (!sc) { + return NS_ERROR_NOT_INITIALIZED; + } + return sc->GetDebugObjectOutputStream(stream, outStream); +} + +nsresult +StartupCacheWrapper::StartupWriteComplete(bool *complete) +{ + StartupCache* sc = StartupCache::GetSingleton(); + if (!sc) { + return NS_ERROR_NOT_INITIALIZED; + } + sc->WaitOnWriteThread(); + *complete = sc->mStartupWriteInitiated && sc->mTable.Count() == 0; + return NS_OK; +} + +nsresult +StartupCacheWrapper::ResetStartupWriteTimer() +{ + StartupCache* sc = StartupCache::GetSingleton(); + return sc ? sc->ResetStartupWriteTimer() : NS_ERROR_NOT_INITIALIZED; +} + +nsresult +StartupCacheWrapper::GetObserver(nsIObserver** obv) { + StartupCache* sc = StartupCache::GetSingleton(); + if (!sc) { + return NS_ERROR_NOT_INITIALIZED; + } + NS_ADDREF(*obv = sc->mListener); + return NS_OK; +} + +nsresult +StartupCacheWrapper::RecordAgesAlways() { + StartupCache *sc = StartupCache::GetSingleton(); + return sc ? sc->RecordAgesAlways() : NS_ERROR_NOT_INITIALIZED; +} + +} // namespace scache +} // namespace mozilla diff --git a/startupcache/StartupCache.h b/startupcache/StartupCache.h new file mode 100644 index 000000000..be428c004 --- /dev/null +++ b/startupcache/StartupCache.h @@ -0,0 +1,230 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2; -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#ifndef StartupCache_h_ +#define StartupCache_h_ + +#include "nsClassHashtable.h" +#include "nsComponentManagerUtils.h" +#include "nsTArray.h" +#include "nsZipArchive.h" +#include "nsIStartupCache.h" +#include "nsITimer.h" +#include "nsIMemoryReporter.h" +#include "nsIObserverService.h" +#include "nsIObserver.h" +#include "nsIOutputStream.h" +#include "nsIFile.h" +#include "mozilla/Attributes.h" +#include "mozilla/MemoryReporting.h" +#include "mozilla/StaticPtr.h" +#include "mozilla/UniquePtr.h" + +/** + * The StartupCache is a persistent cache of simple key-value pairs, + * where the keys are null-terminated c-strings and the values are + * arbitrary data, passed as a (char*, size) tuple. + * + * Clients should use the GetSingleton() static method to access the cache. It + * will be available from the end of XPCOM init (NS_InitXPCOM3 in XPCOMInit.cpp), + * until XPCOM shutdown begins. The GetSingleton() method will return null if the cache + * is unavailable. The cache is only provided for libxul builds -- + * it will fail to link in non-libxul builds. The XPCOM interface is provided + * only to allow compiled-code tests; clients should avoid using it. + * + * The API provided is very simple: GetBuffer() returns a buffer that was previously + * stored in the cache (if any), and PutBuffer() inserts a buffer into the cache. + * GetBuffer returns a new buffer, and the caller must take ownership of it. + * PutBuffer will assert if the client attempts to insert a buffer with the same name as + * an existing entry. The cache makes a copy of the passed-in buffer, so client + * retains ownership. + * + * InvalidateCache() may be called if a client suspects data corruption + * or wishes to invalidate for any other reason. This will remove all existing cache data. + * Additionally, the static method IgnoreDiskCache() can be called if it is + * believed that the on-disk cache file is itself corrupt. This call implicitly + * calls InvalidateCache (if the singleton has been initialized) to ensure any + * data already read from disk is discarded. The cache will not load data from + * the disk file until a successful write occurs. + * + * Finally, getDebugObjectOutputStream() allows debug code to wrap an objectstream + * with a debug objectstream, to check for multiply-referenced objects. These will + * generally fail to deserialize correctly, unless they are stateless singletons or the + * client maintains their own object data map for deserialization. + * + * Writes before the final-ui-startup notification are placed in an intermediate + * cache in memory, then written out to disk at a later time, to get writes off the + * startup path. In any case, clients should not rely on being able to GetBuffer() + * data that is written to the cache, since it may not have been written to disk or + * another client may have invalidated the cache. In other words, it should be used as + * a cache only, and not a reliable persistent store. + * + * Some utility functions are provided in StartupCacheUtils. These functions wrap the + * buffers into object streams, which may be useful for serializing objects. Note + * the above caution about multiply-referenced objects, though -- the streams are just + * as 'dumb' as the underlying buffers about multiply-referenced objects. They just + * provide some convenience in writing out data. + */ + +namespace mozilla { + +namespace scache { + +struct CacheEntry +{ + UniquePtr data; + uint32_t size; + + CacheEntry() : size(0) { } + + // Takes possession of buf + CacheEntry(UniquePtr buf, uint32_t len) : data(Move(buf)), size(len) { } + + ~CacheEntry() + { + } + + size_t SizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) { + return mallocSizeOf(this) + mallocSizeOf(data.get()); + } +}; + +// We don't want to refcount StartupCache, and ObserverService wants to +// refcount its listeners, so we'll let it refcount this instead. +class StartupCacheListener final : public nsIObserver +{ + ~StartupCacheListener() {} + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSIOBSERVER +}; + +class StartupCache : public nsIMemoryReporter +{ + +friend class StartupCacheListener; +friend class StartupCacheWrapper; + +public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSIMEMORYREPORTER + + // StartupCache methods. See above comments for a more detailed description. + + // Returns a buffer that was previously stored, caller takes ownership. + nsresult GetBuffer(const char* id, UniquePtr* outbuf, uint32_t* length); + + // Stores a buffer. Caller keeps ownership, we make a copy. + nsresult PutBuffer(const char* id, const char* inbuf, uint32_t length); + + // Removes the cache file. + void InvalidateCache(); + + // Signal that data should not be loaded from the cache file + static void IgnoreDiskCache(); + + // In DEBUG builds, returns a stream that will attempt to check for + // and disallow multiple writes of the same object. + nsresult GetDebugObjectOutputStream(nsIObjectOutputStream* aStream, + nsIObjectOutputStream** outStream); + + nsresult RecordAgesAlways(); + + static StartupCache* GetSingleton(); + static void DeleteSingleton(); + + // This measures all the heap memory used by the StartupCache, i.e. it + // excludes the mapping. + size_t HeapSizeOfIncludingThis(mozilla::MallocSizeOf mallocSizeOf) const; + + size_t SizeOfMapping(); + +private: + StartupCache(); + virtual ~StartupCache(); + + enum TelemetrifyAge { + IGNORE_AGE = 0, + RECORD_AGE = 1 + }; + static enum TelemetrifyAge gPostFlushAgeAction; + + nsresult LoadArchive(enum TelemetrifyAge flag); + nsresult Init(); + void WriteToDisk(); + nsresult ResetStartupWriteTimer(); + void WaitOnWriteThread(); + + static nsresult InitSingleton(); + static void WriteTimeout(nsITimer *aTimer, void *aClosure); + static void ThreadedWrite(void *aClosure); + + nsClassHashtable mTable; + nsTArray mPendingWrites; + RefPtr mArchive; + nsCOMPtr mFile; + + nsCOMPtr mObserverService; + RefPtr mListener; + nsCOMPtr mTimer; + + bool mStartupWriteInitiated; + + static StaticRefPtr gStartupCache; + static bool gShutdownInitiated; + static bool gIgnoreDiskCache; + PRThread *mWriteThread; +#ifdef DEBUG + nsTHashtable mWriteObjectMap; +#endif +}; + +// This debug outputstream attempts to detect if clients are writing multiple +// references to the same object. We only support that if that object +// is a singleton. +#ifdef DEBUG +class StartupCacheDebugOutputStream final + : public nsIObjectOutputStream +{ + ~StartupCacheDebugOutputStream() {} + + NS_DECL_ISUPPORTS + NS_DECL_NSIOBJECTOUTPUTSTREAM + + StartupCacheDebugOutputStream (nsIObjectOutputStream* binaryStream, + nsTHashtable* objectMap) + : mBinaryStream(binaryStream), mObjectMap(objectMap) { } + + NS_FORWARD_SAFE_NSIBINARYOUTPUTSTREAM(mBinaryStream) + NS_FORWARD_SAFE_NSIOUTPUTSTREAM(mBinaryStream) + + bool CheckReferences(nsISupports* aObject); + + nsCOMPtr mBinaryStream; + nsTHashtable *mObjectMap; +}; +#endif // DEBUG + +// XPCOM wrapper interface provided for tests only. +#define NS_STARTUPCACHE_CID \ + {0xae4505a9, 0x87ab, 0x477c, \ + {0xb5, 0x77, 0xf9, 0x23, 0x57, 0xed, 0xa8, 0x84}} +// contract id: "@mozilla.org/startupcache/cache;1" + +class StartupCacheWrapper final + : public nsIStartupCache +{ + ~StartupCacheWrapper(); + + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSISTARTUPCACHE + + static StartupCacheWrapper* GetSingleton(); + static StartupCacheWrapper *gStartupCacheWrapper; +}; + +} // namespace scache +} // namespace mozilla + +#endif //StartupCache_h_ diff --git a/startupcache/StartupCacheModule.cpp b/startupcache/StartupCacheModule.cpp new file mode 100644 index 000000000..aa9f08e33 --- /dev/null +++ b/startupcache/StartupCacheModule.cpp @@ -0,0 +1,45 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2; -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include + +#include "nscore.h" + +#include "nsID.h" +#include "nsIComponentManager.h" +#include "nsIServiceManager.h" +#include "nsCOMPtr.h" +#include "nsIModule.h" +#include "mozilla/ModuleUtils.h" +#include "mozilla/scache/StartupCache.h" + +using namespace mozilla::scache; + +// XXX Need help with guard for ENABLE_TEST +NS_GENERIC_FACTORY_SINGLETON_CONSTRUCTOR(StartupCacheWrapper, + StartupCacheWrapper::GetSingleton) +NS_DEFINE_NAMED_CID(NS_STARTUPCACHE_CID); + +static const mozilla::Module::CIDEntry kStartupCacheCIDs[] = { + { &kNS_STARTUPCACHE_CID, false, nullptr, StartupCacheWrapperConstructor }, + { nullptr } +}; + +static const mozilla::Module::ContractIDEntry kStartupCacheContracts[] = { + { "@mozilla.org/startupcache/cache;1", &kNS_STARTUPCACHE_CID }, + { nullptr } +}; + +static const mozilla::Module kStartupCacheModule = { + mozilla::Module::kVersion, + kStartupCacheCIDs, + kStartupCacheContracts, + nullptr, + nullptr, + nullptr, + nullptr +}; + +NSMODULE_DEFN(StartupCacheModule) = &kStartupCacheModule; diff --git a/startupcache/StartupCacheUtils.cpp b/startupcache/StartupCacheUtils.cpp new file mode 100644 index 000000000..19a6b4e1b --- /dev/null +++ b/startupcache/StartupCacheUtils.cpp @@ -0,0 +1,253 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsCOMPtr.h" +#include "nsIInputStream.h" +#include "nsIStringStream.h" +#include "nsNetUtil.h" +#include "nsIFileURL.h" +#include "nsIJARURI.h" +#include "nsIResProtocolHandler.h" +#include "nsIChromeRegistry.h" +#include "nsAutoPtr.h" +#include "StartupCacheUtils.h" +#include "mozilla/scache/StartupCache.h" +#include "mozilla/Omnijar.h" + +namespace mozilla { +namespace scache { + +NS_EXPORT nsresult +NewObjectInputStreamFromBuffer(UniquePtr buffer, uint32_t len, + nsIObjectInputStream** stream) +{ + nsCOMPtr stringStream = + do_CreateInstance("@mozilla.org/io/string-input-stream;1"); + NS_ENSURE_TRUE(stringStream, NS_ERROR_FAILURE); + + nsCOMPtr objectInput = + do_CreateInstance("@mozilla.org/binaryinputstream;1"); + NS_ENSURE_TRUE(objectInput, NS_ERROR_FAILURE); + + stringStream->AdoptData(buffer.release(), len); + objectInput->SetInputStream(stringStream); + + objectInput.forget(stream); + return NS_OK; +} + +NS_EXPORT nsresult +NewObjectOutputWrappedStorageStream(nsIObjectOutputStream **wrapperStream, + nsIStorageStream** stream, + bool wantDebugStream) +{ + nsCOMPtr storageStream; + + nsresult rv = NS_NewStorageStream(256, UINT32_MAX, getter_AddRefs(storageStream)); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr objectOutput + = do_CreateInstance("@mozilla.org/binaryoutputstream;1"); + nsCOMPtr outputStream + = do_QueryInterface(storageStream); + + objectOutput->SetOutputStream(outputStream); + +#ifdef DEBUG + if (wantDebugStream) { + // Wrap in debug stream to detect unsupported writes of + // multiply-referenced non-singleton objects + StartupCache* sc = StartupCache::GetSingleton(); + NS_ENSURE_TRUE(sc, NS_ERROR_UNEXPECTED); + nsCOMPtr debugStream; + sc->GetDebugObjectOutputStream(objectOutput, getter_AddRefs(debugStream)); + debugStream.forget(wrapperStream); + } else { + objectOutput.forget(wrapperStream); + } +#else + objectOutput.forget(wrapperStream); +#endif + + storageStream.forget(stream); + return NS_OK; +} + +NS_EXPORT nsresult +NewBufferFromStorageStream(nsIStorageStream *storageStream, + UniquePtr* buffer, uint32_t* len) +{ + nsresult rv; + nsCOMPtr inputStream; + rv = storageStream->NewInputStream(0, getter_AddRefs(inputStream)); + NS_ENSURE_SUCCESS(rv, rv); + + uint64_t avail64; + rv = inputStream->Available(&avail64); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(avail64 <= UINT32_MAX, NS_ERROR_FILE_TOO_BIG); + + uint32_t avail = (uint32_t)avail64; + auto temp = MakeUnique(avail); + uint32_t read; + rv = inputStream->Read(temp.get(), avail, &read); + if (NS_SUCCEEDED(rv) && avail != read) + rv = NS_ERROR_UNEXPECTED; + + if (NS_FAILED(rv)) { + return rv; + } + + *len = avail; + *buffer = Move(temp); + return NS_OK; +} + +static const char baseName[2][5] = { "gre/", "app/" }; + +static inline bool +canonicalizeBase(nsAutoCString &spec, + nsACString &out) +{ + nsAutoCString greBase, appBase; + nsresult rv = mozilla::Omnijar::GetURIString(mozilla::Omnijar::GRE, greBase); + if (NS_FAILED(rv) || !greBase.Length()) + return false; + + rv = mozilla::Omnijar::GetURIString(mozilla::Omnijar::APP, appBase); + if (NS_FAILED(rv)) + return false; + + bool underGre = !greBase.Compare(spec.get(), false, greBase.Length()); + bool underApp = appBase.Length() && + !appBase.Compare(spec.get(), false, appBase.Length()); + + if (!underGre && !underApp) + return false; + + /** + * At this point, if both underGre and underApp are true, it can be one + * of the two following cases: + * - the GRE directory points to a subdirectory of the APP directory, + * meaning spec points under GRE. + * - the APP directory points to a subdirectory of the GRE directory, + * meaning spec points under APP. + * Checking the GRE and APP path length is enough to know in which case + * we are. + */ + if (underGre && underApp && greBase.Length() < appBase.Length()) + underGre = false; + + out.AppendLiteral("/resource/"); + out.Append(baseName[underGre ? mozilla::Omnijar::GRE : mozilla::Omnijar::APP]); + out.Append(Substring(spec, underGre ? greBase.Length() : appBase.Length())); + return true; +} + +/** + * PathifyURI transforms uris into useful zip paths + * to make it easier to manipulate startup cache entries + * using standard zip tools. + * Transformations applied: + * * resource:// URIs are resolved to their corresponding file/jar URI to + * canonicalize resources URIs other than gre and app. + * * Paths under GRE or APP directory have their base path replaced with + * resource/gre or resource/app to avoid depending on install location. + * * jar:file:///path/to/file.jar!/sub/path urls are replaced with + * /path/to/file.jar/sub/path + * + * The result is appended to the string passed in. Adding a prefix before + * calling is recommended to avoid colliding with other cache users. + * + * For example, in the js loader (string is prefixed with jsloader by caller): + * resource://gre/modules/XPCOMUtils.jsm or + * file://$GRE_DIR/modules/XPCOMUtils.jsm or + * jar:file://$GRE_DIR/omni.jar!/modules/XPCOMUtils.jsm becomes + * jsloader/resource/gre/modules/XPCOMUtils.jsm + * file://$PROFILE_DIR/extensions/{uuid}/components/component.js becomes + * jsloader/$PROFILE_DIR/extensions/%7Buuid%7D/components/component.js + * jar:file://$PROFILE_DIR/extensions/some.xpi!/components/component.js becomes + * jsloader/$PROFILE_DIR/extensions/some.xpi/components/component.js + */ +NS_EXPORT nsresult +PathifyURI(nsIURI *in, nsACString &out) +{ + bool equals; + nsresult rv; + nsCOMPtr uri = in; + nsAutoCString spec; + + // Resolve resource:// URIs. At the end of this if/else block, we + // have both spec and uri variables identifying the same URI. + if (NS_SUCCEEDED(in->SchemeIs("resource", &equals)) && equals) { + nsCOMPtr ioService = do_GetIOService(&rv); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr ph; + rv = ioService->GetProtocolHandler("resource", getter_AddRefs(ph)); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr irph(do_QueryInterface(ph, &rv)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = irph->ResolveURI(in, spec); + NS_ENSURE_SUCCESS(rv, rv); + + rv = ioService->NewURI(spec, nullptr, nullptr, getter_AddRefs(uri)); + NS_ENSURE_SUCCESS(rv, rv); + } else { + if (NS_SUCCEEDED(in->SchemeIs("chrome", &equals)) && equals) { + nsCOMPtr chromeReg = + mozilla::services::GetChromeRegistryService(); + if (!chromeReg) + return NS_ERROR_UNEXPECTED; + + rv = chromeReg->ConvertChromeURL(in, getter_AddRefs(uri)); + NS_ENSURE_SUCCESS(rv, rv); + } + + rv = uri->GetSpec(spec); + NS_ENSURE_SUCCESS(rv, rv); + } + + if (!canonicalizeBase(spec, out)) { + if (NS_SUCCEEDED(uri->SchemeIs("file", &equals)) && equals) { + nsCOMPtr baseFileURL; + baseFileURL = do_QueryInterface(uri, &rv); + NS_ENSURE_SUCCESS(rv, rv); + + nsAutoCString path; + rv = baseFileURL->GetPath(path); + NS_ENSURE_SUCCESS(rv, rv); + + out.Append(path); + } else if (NS_SUCCEEDED(uri->SchemeIs("jar", &equals)) && equals) { + nsCOMPtr jarURI = do_QueryInterface(uri, &rv); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr jarFileURI; + rv = jarURI->GetJARFile(getter_AddRefs(jarFileURI)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = PathifyURI(jarFileURI, out); + NS_ENSURE_SUCCESS(rv, rv); + + nsAutoCString path; + rv = jarURI->GetJAREntry(path); + NS_ENSURE_SUCCESS(rv, rv); + out.Append('/'); + out.Append(path); + } else { // Very unlikely + rv = uri->GetSpec(spec); + NS_ENSURE_SUCCESS(rv, rv); + + out.Append('/'); + out.Append(spec); + } + } + return NS_OK; +} + +} // namespace scache +} // namespace mozilla diff --git a/startupcache/StartupCacheUtils.h b/startupcache/StartupCacheUtils.h new file mode 100644 index 000000000..7a4da2543 --- /dev/null +++ b/startupcache/StartupCacheUtils.h @@ -0,0 +1,43 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2; -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ +#ifndef nsStartupCacheUtils_h_ +#define nsStartupCacheUtils_h_ + +#include "nsIStorageStream.h" +#include "nsIObjectInputStream.h" +#include "nsIObjectOutputStream.h" +#include "mozilla/UniquePtr.h" + +namespace mozilla { +namespace scache { + +NS_EXPORT nsresult +NewObjectInputStreamFromBuffer(UniquePtr buffer, uint32_t len, + nsIObjectInputStream** stream); + +// We can't retrieve the wrapped stream from the objectOutputStream later, +// so we return it here. We give callers in debug builds the option +// to wrap the outputstream in a debug stream, which will detect if +// non-singleton objects are written out multiple times during a serialization. +// This could cause them to be deserialized incorrectly (as multiple copies +// instead of references). +NS_EXPORT nsresult +NewObjectOutputWrappedStorageStream(nsIObjectOutputStream **wrapperStream, + nsIStorageStream** stream, + bool wantDebugStream); + +// Creates a buffer for storing the stream into the cache. The buffer is +// allocated with 'new []'. After calling this function, the caller would +// typically call nsIStartupCache::PutBuffer with the returned buffer. +NS_EXPORT nsresult +NewBufferFromStorageStream(nsIStorageStream *storageStream, + UniquePtr* buffer, uint32_t* len); + +NS_EXPORT nsresult +PathifyURI(nsIURI *in, nsACString &out); +} // namespace scache +} // namespace mozilla + +#endif //nsStartupCacheUtils_h_ diff --git a/startupcache/moz.build b/startupcache/moz.build new file mode 100644 index 000000000..24068cb7e --- /dev/null +++ b/startupcache/moz.build @@ -0,0 +1,28 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +if not CONFIG['MOZ_B2G']: + TEST_DIRS += ['test'] + +XPIDL_SOURCES += [ + 'nsIStartupCache.idl', +] + +XPIDL_MODULE = 'startupcache' + +EXPORTS.mozilla.scache += [ + 'StartupCache.h', + 'StartupCacheUtils.h', +] + +# These files cannot be built in unified mode because they rely on plarena.h. +SOURCES += [ + 'StartupCache.cpp', + 'StartupCacheModule.cpp', + 'StartupCacheUtils.cpp', +] + +FINAL_LIBRARY = 'xul' diff --git a/startupcache/nsIStartupCache.idl b/startupcache/nsIStartupCache.idl new file mode 100644 index 000000000..b03dcb3cc --- /dev/null +++ b/startupcache/nsIStartupCache.idl @@ -0,0 +1,65 @@ +/* -*- Mode: IDL; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- + * + * This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsIInputStream.idl" +#include "nsISupports.idl" +#include "nsIObserver.idl" +#include "nsIObjectOutputStream.idl" + +%{C++ +#include "mozilla/UniquePtr.h" +%} + +[uuid(25957820-90a1-428c-8739-b0845d3cc534)] +interface nsIStartupCache : nsISupports +{ + + /** This interface is provided for testing purposes only, basically + * just to solve link vagaries. See docs in StartupCache.h + * GetBuffer, PutBuffer, and InvalidateCache act as described + * in that file. */ + + uint32_t getBuffer(in string aID, out charPtr aBuffer); +%{C++ + /* A more convenient interface for using from C++. */ + nsresult GetBuffer(const char* id, mozilla::UniquePtr* outbuf, uint32_t* length) + { + char* buf; + nsresult rv = GetBuffer(id, &buf, length); + NS_ENSURE_SUCCESS(rv, rv); + outbuf->reset(buf); + return rv; + } +%} + + void putBuffer(in string aID, in string aBuffer, + in uint32_t aLength); + + void invalidateCache(); + + void ignoreDiskCache(); + + /** In debug builds, wraps this object output stream with a stream that will + * detect and prevent the write of a multiply-referenced non-singleton object + * during serialization. In non-debug, returns an add-ref'd pointer to + * original stream, unwrapped. */ + nsIObjectOutputStream getDebugObjectOutputStream(in nsIObjectOutputStream aStream); + + /* Allows clients to check whether the one-time writeout after startup + * has finished yet, and also to set this variable as needed (so test + * code can fire mulitple startup writes if needed). + */ + boolean startupWriteComplete(); + void resetStartupWriteTimer(); + + /* Instruct clients to always post cache ages to Telemetry, even in + cases where it would not normally make sense. */ + void recordAgesAlways(); + + /* Allows clients to simulate the behavior of ObserverService. */ + readonly attribute nsIObserver observer; +}; + diff --git a/startupcache/test/TestStartupCache.cpp b/startupcache/test/TestStartupCache.cpp new file mode 100644 index 000000000..a16c2de72 --- /dev/null +++ b/startupcache/test/TestStartupCache.cpp @@ -0,0 +1,480 @@ +/* -*- Mode: C++; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2; -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "TestHarness.h" + +#include "nsThreadUtils.h" +#include "nsIClassInfo.h" +#include "nsIOutputStream.h" +#include "nsIObserver.h" +#include "nsISerializable.h" +#include "nsISupports.h" +#include "nsIStartupCache.h" +#include "nsIStringStream.h" +#include "nsIStorageStream.h" +#include "nsIObjectInputStream.h" +#include "nsIObjectOutputStream.h" +#include "nsIURI.h" +#include "nsStringAPI.h" +#include "nsIPrefBranch.h" +#include "nsIPrefService.h" +#include "nsIXPConnect.h" +#include "prio.h" +#include "mozilla/Maybe.h" +#include "mozilla/UniquePtr.h" + +using namespace JS; + +namespace mozilla { +namespace scache { + +NS_IMPORT nsresult +NewObjectInputStreamFromBuffer(UniquePtr buffer, uint32_t len, + nsIObjectInputStream** stream); + +// We can't retrieve the wrapped stream from the objectOutputStream later, +// so we return it here. +NS_IMPORT nsresult +NewObjectOutputWrappedStorageStream(nsIObjectOutputStream **wrapperStream, + nsIStorageStream** stream); + +NS_IMPORT nsresult +NewBufferFromStorageStream(nsIStorageStream *storageStream, + UniquePtr* buffer, uint32_t* len); +} // namespace scache +} // namespace mozilla + +using namespace mozilla::scache; +using mozilla::UniquePtr; + +#define NS_ENSURE_STR_MATCH(str1, str2, testname) \ +PR_BEGIN_MACRO \ +if (0 != strcmp(str1, str2)) { \ + fail("failed " testname); \ + return NS_ERROR_FAILURE; \ +} \ +passed("passed " testname); \ +PR_END_MACRO + +nsresult +WaitForStartupTimer() { + nsresult rv; + nsCOMPtr sc + = do_GetService("@mozilla.org/startupcache/cache;1"); + PR_Sleep(10 * PR_TicksPerSecond()); + + bool complete; + while (true) { + + NS_ProcessPendingEvents(nullptr); + rv = sc->StartupWriteComplete(&complete); + if (NS_FAILED(rv) || complete) + break; + PR_Sleep(1 * PR_TicksPerSecond()); + } + return rv; +} + +nsresult +TestStartupWriteRead() { + nsresult rv; + nsCOMPtr sc + = do_GetService("@mozilla.org/startupcache/cache;1", &rv); + if (!sc) { + fail("didn't get a pointer..."); + return NS_ERROR_FAILURE; + } else { + passed("got a pointer?"); + } + sc->InvalidateCache(); + + const char* buf = "Market opportunities for BeardBook"; + const char* id = "id"; + UniquePtr outbuf; + uint32_t len; + + rv = sc->PutBuffer(id, buf, strlen(buf) + 1); + NS_ENSURE_SUCCESS(rv, rv); + + rv = sc->GetBuffer(id, &outbuf, &len); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_STR_MATCH(buf, outbuf.get(), "pre-write read"); + + rv = sc->ResetStartupWriteTimer(); + rv = WaitForStartupTimer(); + NS_ENSURE_SUCCESS(rv, rv); + + rv = sc->GetBuffer(id, &outbuf, &len); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_STR_MATCH(buf, outbuf.get(), "simple write/read"); + + return NS_OK; +} + +nsresult +TestWriteInvalidateRead() { + nsresult rv; + const char* buf = "BeardBook competitive analysis"; + const char* id = "id"; + UniquePtr outbuf; + uint32_t len; + nsCOMPtr sc + = do_GetService("@mozilla.org/startupcache/cache;1", &rv); + sc->InvalidateCache(); + + rv = sc->PutBuffer(id, buf, strlen(buf) + 1); + NS_ENSURE_SUCCESS(rv, rv); + + sc->InvalidateCache(); + + rv = sc->GetBuffer(id, &outbuf, &len); + if (rv == NS_ERROR_NOT_AVAILABLE) { + passed("buffer not available after invalidate"); + } else if (NS_SUCCEEDED(rv)) { + fail("GetBuffer succeeded unexpectedly after invalidate"); + return NS_ERROR_UNEXPECTED; + } else { + fail("GetBuffer gave an unexpected failure, expected NOT_AVAILABLE"); + return rv; + } + + sc->InvalidateCache(); + return NS_OK; +} + +nsresult +TestWriteObject() { + nsresult rv; + + nsCOMPtr obj + = do_CreateInstance("@mozilla.org/network/simple-uri;1"); + if (!obj) { + fail("did not create object in test write object"); + return NS_ERROR_UNEXPECTED; + } + NS_NAMED_LITERAL_CSTRING(spec, "http://www.mozilla.org"); + rv = obj->SetSpec(spec); + NS_ENSURE_SUCCESS(rv, rv); + nsCOMPtr sc = do_GetService("@mozilla.org/startupcache/cache;1", &rv); + + sc->InvalidateCache(); + + // Create an object stream. Usually this is done with + // NewObjectOutputWrappedStorageStream, but that uses + // StartupCache::GetSingleton in debug builds, and we + // don't have access to that here. Obviously. + const char* id = "id"; + nsCOMPtr storageStream + = do_CreateInstance("@mozilla.org/storagestream;1"); + NS_ENSURE_ARG_POINTER(storageStream); + + rv = storageStream->Init(256, (uint32_t) -1); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr objectOutput + = do_CreateInstance("@mozilla.org/binaryoutputstream;1"); + if (!objectOutput) + return NS_ERROR_OUT_OF_MEMORY; + + nsCOMPtr outputStream + = do_QueryInterface(storageStream); + + rv = objectOutput->SetOutputStream(outputStream); + + if (NS_FAILED(rv)) { + fail("failed to create output stream"); + return rv; + } + nsCOMPtr objQI(do_QueryInterface(obj)); + rv = objectOutput->WriteObject(objQI, true); + if (NS_FAILED(rv)) { + fail("failed to write object"); + return rv; + } + + UniquePtr buf; + uint32_t len; + NewBufferFromStorageStream(storageStream, &buf, &len); + + // Since this is a post-startup write, it should be written and + // available. + rv = sc->PutBuffer(id, buf.get(), len); + if (NS_FAILED(rv)) { + fail("failed to insert input stream"); + return rv; + } + + UniquePtr buf2; + uint32_t len2; + nsCOMPtr objectInput; + rv = sc->GetBuffer(id, &buf2, &len2); + if (NS_FAILED(rv)) { + fail("failed to retrieve buffer"); + return rv; + } + + rv = NewObjectInputStreamFromBuffer(Move(buf2), len2, + getter_AddRefs(objectInput)); + if (NS_FAILED(rv)) { + fail("failed to created input stream"); + return rv; + } + + nsCOMPtr deserialized; + rv = objectInput->ReadObject(true, getter_AddRefs(deserialized)); + if (NS_FAILED(rv)) { + fail("failed to read object"); + return rv; + } + + bool match = false; + nsCOMPtr uri(do_QueryInterface(deserialized)); + if (uri) { + nsCString outSpec; + rv = uri->GetSpec(outSpec); + if (NS_FAILED(rv)) { + fail("failed to get spec"); + return rv; + } + match = outSpec.Equals(spec); + } + if (!match) { + fail("deserialized object has incorrect information"); + return rv; + } + + passed("write object"); + return NS_OK; +} + +nsresult +LockCacheFile(bool protect, nsIFile* profileDir) { + NS_ENSURE_ARG(profileDir); + + nsCOMPtr startupCache; + profileDir->Clone(getter_AddRefs(startupCache)); + NS_ENSURE_STATE(startupCache); + startupCache->AppendNative(NS_LITERAL_CSTRING("startupCache")); + + nsresult rv; +#ifndef XP_WIN + static uint32_t oldPermissions; +#else + static PRFileDesc* fd = nullptr; +#endif + + // To prevent deletion of the startupcache file, we change the containing + // directory's permissions on Linux/Mac, and hold the file open on Windows + if (protect) { +#ifndef XP_WIN + rv = startupCache->GetPermissions(&oldPermissions); + NS_ENSURE_SUCCESS(rv, rv); + rv = startupCache->SetPermissions(0555); + NS_ENSURE_SUCCESS(rv, rv); +#else + // Filename logic from StartupCache.cpp + #ifdef IS_BIG_ENDIAN + #define SC_ENDIAN "big" + #else + #define SC_ENDIAN "little" + #endif + + #if PR_BYTES_PER_WORD == 4 + #define SC_WORDSIZE "4" + #else + #define SC_WORDSIZE "8" + #endif + char sStartupCacheName[] = "startupCache." SC_WORDSIZE "." SC_ENDIAN; + startupCache->AppendNative(NS_LITERAL_CSTRING(sStartupCacheName)); + + rv = startupCache->OpenNSPRFileDesc(PR_RDONLY, 0, &fd); + NS_ENSURE_SUCCESS(rv, rv); +#endif + } else { +#ifndef XP_WIN + rv = startupCache->SetPermissions(oldPermissions); + NS_ENSURE_SUCCESS(rv, rv); +#else + PR_Close(fd); +#endif + } + + return NS_OK; +} + +nsresult +TestIgnoreDiskCache(nsIFile* profileDir) { + nsresult rv; + nsCOMPtr sc + = do_GetService("@mozilla.org/startupcache/cache;1", &rv); + sc->InvalidateCache(); + + const char* buf = "Get a Beardbook app for your smartphone"; + const char* id = "id"; + UniquePtr outbuf; + uint32_t len; + + rv = sc->PutBuffer(id, buf, strlen(buf) + 1); + NS_ENSURE_SUCCESS(rv, rv); + rv = sc->ResetStartupWriteTimer(); + rv = WaitForStartupTimer(); + NS_ENSURE_SUCCESS(rv, rv); + + // Prevent StartupCache::InvalidateCache from deleting the disk file + rv = LockCacheFile(true, profileDir); + NS_ENSURE_SUCCESS(rv, rv); + + sc->IgnoreDiskCache(); + + rv = sc->GetBuffer(id, &outbuf, &len); + + nsresult r = LockCacheFile(false, profileDir); + NS_ENSURE_SUCCESS(r, r); + + if (rv == NS_ERROR_NOT_AVAILABLE) { + passed("buffer not available after ignoring disk cache"); + } else if (NS_SUCCEEDED(rv)) { + fail("GetBuffer succeeded unexpectedly after ignoring disk cache"); + return NS_ERROR_UNEXPECTED; + } else { + fail("GetBuffer gave an unexpected failure, expected NOT_AVAILABLE"); + return rv; + } + + sc->InvalidateCache(); + return NS_OK; +} + +nsresult +TestEarlyShutdown() { + nsresult rv; + nsCOMPtr sc + = do_GetService("@mozilla.org/startupcache/cache;1", &rv); + sc->InvalidateCache(); + + const char* buf = "Find your soul beardmate on BeardBook"; + const char* id = "id"; + uint32_t len; + UniquePtr outbuf; + + sc->ResetStartupWriteTimer(); + rv = sc->PutBuffer(id, buf, strlen(buf) + 1); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr obs; + sc->GetObserver(getter_AddRefs(obs)); + obs->Observe(nullptr, "xpcom-shutdown", nullptr); + rv = WaitForStartupTimer(); + NS_ENSURE_SUCCESS(rv, rv); + + rv = sc->GetBuffer(id, &outbuf, &len); + + if (NS_SUCCEEDED(rv)) { + passed("GetBuffer succeeded after early shutdown"); + } else { + fail("GetBuffer failed after early shutdown"); + return rv; + } + + const char* other_id = "other_id"; + rv = sc->PutBuffer(other_id, buf, strlen(buf) + 1); + + if (rv == NS_ERROR_NOT_AVAILABLE) { + passed("PutBuffer not available after early shutdown"); + } else if (NS_SUCCEEDED(rv)) { + fail("PutBuffer succeeded unexpectedly after early shutdown"); + return NS_ERROR_UNEXPECTED; + } else { + fail("PutBuffer gave an unexpected failure, expected NOT_AVAILABLE"); + return rv; + } + + return NS_OK; +} + +int main(int argc, char** argv) +{ + ScopedXPCOM xpcom("Startup Cache"); + if (xpcom.failed()) + return 1; + + nsCOMPtr prefs = do_GetService(NS_PREFSERVICE_CONTRACTID); + if (!prefs) { + fail("prefs"); + return 1; + } + prefs->SetIntPref("hangmonitor.timeout", 0); + + int rv = 0; + nsresult scrv; + + // Register TestStartupCacheTelemetry + nsCOMPtr manifest; + scrv = NS_GetSpecialDirectory(NS_GRE_DIR, + getter_AddRefs(manifest)); + if (NS_FAILED(scrv)) { + fail("NS_XPCOM_CURRENT_PROCESS_DIR"); + return 1; + } + +#ifdef XP_MACOSX + nsCOMPtr tempManifest; + manifest->Clone(getter_AddRefs(tempManifest)); + manifest->AppendNative( + NS_LITERAL_CSTRING("TestStartupCacheTelemetry.manifest")); + bool exists; + manifest->Exists(&exists); + if (!exists) { + // Workaround for bug 1080338 in mozharness. + manifest = tempManifest.forget(); + manifest->SetNativeLeafName(NS_LITERAL_CSTRING("MacOS")); + manifest->AppendNative( + NS_LITERAL_CSTRING("TestStartupCacheTelemetry.manifest")); + } +#else + manifest->AppendNative( + NS_LITERAL_CSTRING("TestStartupCacheTelemetry.manifest")); +#endif + + XRE_AddManifestLocation(NS_APP_LOCATION, manifest); + + nsCOMPtr telemetryThing = + do_GetService("@mozilla.org/testing/startup-cache-telemetry.js"); + if (!telemetryThing) { + fail("telemetryThing"); + return 1; + } + scrv = telemetryThing->Observe(nullptr, "save-initial", nullptr); + if (NS_FAILED(scrv)) { + fail("save-initial"); + rv = 1; + } + + nsCOMPtr sc + = do_GetService("@mozilla.org/startupcache/cache;1", &scrv); + if (NS_FAILED(scrv)) + rv = 1; + else + sc->RecordAgesAlways(); + if (NS_FAILED(TestStartupWriteRead())) + rv = 1; + if (NS_FAILED(TestWriteInvalidateRead())) + rv = 1; + if (NS_FAILED(TestWriteObject())) + rv = 1; + nsCOMPtr profileDir = xpcom.GetProfileDirectory(); + if (NS_FAILED(TestIgnoreDiskCache(profileDir))) + rv = 1; + if (NS_FAILED(TestEarlyShutdown())) + rv = 1; + + scrv = telemetryThing->Observe(nullptr, "save-initial", nullptr); + if (NS_FAILED(scrv)) { + fail("check-final"); + rv = 1; + } + + return rv; +} diff --git a/startupcache/test/TestStartupCacheTelemetry.js b/startupcache/test/TestStartupCacheTelemetry.js new file mode 100644 index 000000000..7a570187f --- /dev/null +++ b/startupcache/test/TestStartupCacheTelemetry.js @@ -0,0 +1,60 @@ +const Cc = Components.classes; +const Ci = Components.interfaces; +const Cu = Components.utils; + +Cu.import("resource://gre/modules/Services.jsm"); +Cu.import("resource://gre/modules/XPCOMUtils.jsm"); + +function shouldHaveChanged(a, b) +{ + if (a.length != b.length) { + throw Error("TEST-UNEXPECTED-FAIL: telemetry count array size changed"); + } + + for (let i = 0; i < a.length; ++i) { + if (a[i] == b[i]) { + continue; + } + return; // something was different, that's all that matters + } + throw Error("TEST-UNEXPECTED-FAIL: telemetry data didn't change"); +} + +function TestStartupCacheTelemetry() { } + +TestStartupCacheTelemetry.prototype = { + classID: Components.ID("{73cbeffd-d6c7-42f0-aaf3-f176430dcfc8}"), + QueryInterface: XPCOMUtils.generateQI([Ci.nsIObserver]), + + saveInitial: function() { + let t = Services.telemetry; + this._age = t.getHistogramById("STARTUP_CACHE_AGE_HOURS").snapshot.counts; + this._invalid = t.getHistogramById("STARTUP_CACHE_INVALID").snapshot.counts; + }, + + checkFinal: function() { + let t = Services.telemetry; + let newAge = t.getHistogramById("STARTUP_CACHE_AGE_HOURS").snapshot.counts; + shouldHaveChanged(this._age, newAge); + + let newInvalid = t.getHistogramById("STARTUP_CACHE_INVALID").snapshot.counts; + shouldHaveChanged(this._invalid, newInvalid); + }, + + observe: function(subject, topic, data) { + switch (topic) { + case "save-initial": + this.saveInitial(); + break; + + case "check-final": + this.checkFinal(); + break; + + default: + throw Error("BADDOG, NO MILKBONE FOR YOU"); + } + }, +}; + +this.NSGetFactory = XPCOMUtils.generateNSGetFactory([TestStartupCacheTelemetry]); diff --git a/startupcache/test/TestStartupCacheTelemetry.manifest b/startupcache/test/TestStartupCacheTelemetry.manifest new file mode 100644 index 000000000..b288a7292 --- /dev/null +++ b/startupcache/test/TestStartupCacheTelemetry.manifest @@ -0,0 +1,2 @@ +component {73cbeffd-d6c7-42f0-aaf3-f176430dcfc8} TestStartupCacheTelemetry.js +contract @mozilla.org/testing/startup-cache-telemetry.js {73cbeffd-d6c7-42f0-aaf3-f176430dcfc8} diff --git a/startupcache/test/moz.build b/startupcache/test/moz.build new file mode 100644 index 000000000..5a9e6a61e --- /dev/null +++ b/startupcache/test/moz.build @@ -0,0 +1,14 @@ +# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*- +# vim: set filetype=python: +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +GeckoCppUnitTests([ + 'TestStartupCache', +]) + +EXTRA_COMPONENTS += [ + 'TestStartupCacheTelemetry.js', + 'TestStartupCacheTelemetry.manifest', +] -- cgit v1.2.3