From 5f8de423f190bbb79a62f804151bc24824fa32d8 Mon Sep 17 00:00:00 2001 From: "Matt A. Tobin" Date: Fri, 2 Feb 2018 04:16:08 -0500 Subject: Add m-esr52 at 52.6.0 --- netwerk/base/Predictor.cpp | 2550 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 2550 insertions(+) create mode 100644 netwerk/base/Predictor.cpp (limited to 'netwerk/base/Predictor.cpp') diff --git a/netwerk/base/Predictor.cpp b/netwerk/base/Predictor.cpp new file mode 100644 index 000000000..e97b11d16 --- /dev/null +++ b/netwerk/base/Predictor.cpp @@ -0,0 +1,2550 @@ +/* vim: set ts=2 sts=2 et sw=2: */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include + +#include "Predictor.h" + +#include "nsAppDirectoryServiceDefs.h" +#include "nsICacheStorage.h" +#include "nsICacheStorageService.h" +#include "nsICachingChannel.h" +#include "nsICancelable.h" +#include "nsIChannel.h" +#include "nsContentUtils.h" +#include "nsIDNSService.h" +#include "nsIDocument.h" +#include "nsIFile.h" +#include "nsIHttpChannel.h" +#include "nsIInputStream.h" +#include "nsIIOService.h" +#include "nsILoadContext.h" +#include "nsILoadContextInfo.h" +#include "nsILoadGroup.h" +#include "nsINetworkPredictorVerifier.h" +#include "nsIObserverService.h" +#include "nsIPrefBranch.h" +#include "nsIPrefService.h" +#include "nsISpeculativeConnect.h" +#include "nsITimer.h" +#include "nsIURI.h" +#include "nsNetUtil.h" +#include "nsServiceManagerUtils.h" +#include "nsStreamUtils.h" +#include "nsString.h" +#include "nsThreadUtils.h" +#include "mozilla/Logging.h" + +#include "mozilla/Preferences.h" +#include "mozilla/Telemetry.h" + +#include "mozilla/net/NeckoCommon.h" +#include "mozilla/net/NeckoParent.h" + +#include "LoadContextInfo.h" +#include "mozilla/ipc/URIUtils.h" +#include "SerializedLoadContext.h" +#include "mozilla/net/NeckoChild.h" + +#include "mozilla/dom/ContentParent.h" + +using namespace mozilla; + +namespace mozilla { +namespace net { + +Predictor *Predictor::sSelf = nullptr; + +static LazyLogModule gPredictorLog("NetworkPredictor"); + +#define PREDICTOR_LOG(args) MOZ_LOG(gPredictorLog, mozilla::LogLevel::Debug, args) + +#define RETURN_IF_FAILED(_rv) \ + do { \ + if (NS_FAILED(_rv)) { \ + return; \ + } \ + } while (0) + +#define NOW_IN_SECONDS() static_cast(PR_Now() / PR_USEC_PER_SEC) + + +static const char PREDICTOR_ENABLED_PREF[] = "network.predictor.enabled"; +static const char PREDICTOR_SSL_HOVER_PREF[] = "network.predictor.enable-hover-on-ssl"; +static const char PREDICTOR_PREFETCH_PREF[] = "network.predictor.enable-prefetch"; + +static const char PREDICTOR_PAGE_DELTA_DAY_PREF[] = + "network.predictor.page-degradation.day"; +static const int32_t PREDICTOR_PAGE_DELTA_DAY_DEFAULT = 0; +static const char PREDICTOR_PAGE_DELTA_WEEK_PREF[] = + "network.predictor.page-degradation.week"; +static const int32_t PREDICTOR_PAGE_DELTA_WEEK_DEFAULT = 5; +static const char PREDICTOR_PAGE_DELTA_MONTH_PREF[] = + "network.predictor.page-degradation.month"; +static const int32_t PREDICTOR_PAGE_DELTA_MONTH_DEFAULT = 10; +static const char PREDICTOR_PAGE_DELTA_YEAR_PREF[] = + "network.predictor.page-degradation.year"; +static const int32_t PREDICTOR_PAGE_DELTA_YEAR_DEFAULT = 25; +static const char PREDICTOR_PAGE_DELTA_MAX_PREF[] = + "network.predictor.page-degradation.max"; +static const int32_t PREDICTOR_PAGE_DELTA_MAX_DEFAULT = 50; +static const char PREDICTOR_SUB_DELTA_DAY_PREF[] = + "network.predictor.subresource-degradation.day"; +static const int32_t PREDICTOR_SUB_DELTA_DAY_DEFAULT = 1; +static const char PREDICTOR_SUB_DELTA_WEEK_PREF[] = + "network.predictor.subresource-degradation.week"; +static const int32_t PREDICTOR_SUB_DELTA_WEEK_DEFAULT = 10; +static const char PREDICTOR_SUB_DELTA_MONTH_PREF[] = + "network.predictor.subresource-degradation.month"; +static const int32_t PREDICTOR_SUB_DELTA_MONTH_DEFAULT = 25; +static const char PREDICTOR_SUB_DELTA_YEAR_PREF[] = + "network.predictor.subresource-degradation.year"; +static const int32_t PREDICTOR_SUB_DELTA_YEAR_DEFAULT = 50; +static const char PREDICTOR_SUB_DELTA_MAX_PREF[] = + "network.predictor.subresource-degradation.max"; +static const int32_t PREDICTOR_SUB_DELTA_MAX_DEFAULT = 100; + +static const char PREDICTOR_PREFETCH_ROLLING_LOAD_PREF[] = + "network.predictor.prefetch-rolling-load-count"; +static const int32_t PREFETCH_ROLLING_LOAD_DEFAULT = 10; +static const char PREDICTOR_PREFETCH_MIN_PREF[] = + "network.predictor.prefetch-min-confidence"; +static const int32_t PREFETCH_MIN_DEFAULT = 100; +static const char PREDICTOR_PRECONNECT_MIN_PREF[] = + "network.predictor.preconnect-min-confidence"; +static const int32_t PRECONNECT_MIN_DEFAULT = 90; +static const char PREDICTOR_PRERESOLVE_MIN_PREF[] = + "network.predictor.preresolve-min-confidence"; +static const int32_t PRERESOLVE_MIN_DEFAULT = 60; +static const char PREDICTOR_REDIRECT_LIKELY_PREF[] = + "network.predictor.redirect-likely-confidence"; +static const int32_t REDIRECT_LIKELY_DEFAULT = 75; + +static const char PREDICTOR_PREFETCH_FORCE_VALID_PREF[] = + "network.predictor.prefetch-force-valid-for"; +static const int32_t PREFETCH_FORCE_VALID_DEFAULT = 10; + +static const char PREDICTOR_MAX_RESOURCES_PREF[] = + "network.predictor.max-resources-per-entry"; +static const uint32_t PREDICTOR_MAX_RESOURCES_DEFAULT = 100; + +// This is selected in concert with max-resources-per-entry to keep memory usage +// low-ish. The default of the combo of the two is ~50k +static const char PREDICTOR_MAX_URI_LENGTH_PREF[] = + "network.predictor.max-uri-length"; +static const uint32_t PREDICTOR_MAX_URI_LENGTH_DEFAULT = 500; + +static const char PREDICTOR_DOING_TESTS_PREF[] = "network.predictor.doing-tests"; + +static const char PREDICTOR_CLEANED_UP_PREF[] = "network.predictor.cleaned-up"; + +// All these time values are in sec +static const uint32_t ONE_DAY = 86400U; +static const uint32_t ONE_WEEK = 7U * ONE_DAY; +static const uint32_t ONE_MONTH = 30U * ONE_DAY; +static const uint32_t ONE_YEAR = 365U * ONE_DAY; + +static const uint32_t STARTUP_WINDOW = 5U * 60U; // 5min + +// Version of metadata entries we expect +static const uint32_t METADATA_VERSION = 1; + +// Flags available in entries +// FLAG_PREFETCHABLE - we have determined that this item is eligible for prefetch +static const uint32_t FLAG_PREFETCHABLE = 1 << 0; + +// We save 12 bits in the "flags" section of our metadata for actual flags, the +// rest are to keep track of a rolling count of which loads a resource has been +// used on to determine if we can prefetch that resource or not; +static const uint8_t kRollingLoadOffset = 12; +static const int32_t kMaxPrefetchRollingLoadCount = 20; +static const uint32_t kFlagsMask = ((1 << kRollingLoadOffset) - 1); + +// ID Extensions for cache entries +#define PREDICTOR_ORIGIN_EXTENSION "predictor-origin" + +// Get the full origin (scheme, host, port) out of a URI (maybe should be part +// of nsIURI instead?) +static nsresult +ExtractOrigin(nsIURI *uri, nsIURI **originUri, nsIIOService *ioService) +{ + nsAutoCString s; + s.Truncate(); + nsresult rv = nsContentUtils::GetASCIIOrigin(uri, s); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_NewURI(originUri, s, nullptr, nullptr, ioService); +} + +// All URIs we get passed *must* be http or https if they're not null. This +// helps ensure that. +static bool +IsNullOrHttp(nsIURI *uri) +{ + if (!uri) { + return true; + } + + bool isHTTP = false; + uri->SchemeIs("http", &isHTTP); + if (!isHTTP) { + uri->SchemeIs("https", &isHTTP); + } + + return isHTTP; +} + +// Listener for the speculative DNS requests we'll fire off, which just ignores +// the result (since we're just trying to warm the cache). This also exists to +// reduce round-trips to the main thread, by being something threadsafe the +// Predictor can use. + +NS_IMPL_ISUPPORTS(Predictor::DNSListener, nsIDNSListener); + +NS_IMETHODIMP +Predictor::DNSListener::OnLookupComplete(nsICancelable *request, + nsIDNSRecord *rec, + nsresult status) +{ + return NS_OK; +} + +// Class to proxy important information from the initial predictor call through +// the cache API and back into the internals of the predictor. We can't use the +// predictor itself, as it may have multiple actions in-flight, and each action +// has different parameters. +NS_IMPL_ISUPPORTS(Predictor::Action, nsICacheEntryOpenCallback); + +Predictor::Action::Action(bool fullUri, bool predict, + Predictor::Reason reason, + nsIURI *targetURI, nsIURI *sourceURI, + nsINetworkPredictorVerifier *verifier, + Predictor *predictor) + :mFullUri(fullUri) + ,mPredict(predict) + ,mTargetURI(targetURI) + ,mSourceURI(sourceURI) + ,mVerifier(verifier) + ,mStackCount(0) + ,mPredictor(predictor) +{ + mStartTime = TimeStamp::Now(); + if (mPredict) { + mPredictReason = reason.mPredict; + } else { + mLearnReason = reason.mLearn; + } +} + +Predictor::Action::Action(bool fullUri, bool predict, + Predictor::Reason reason, + nsIURI *targetURI, nsIURI *sourceURI, + nsINetworkPredictorVerifier *verifier, + Predictor *predictor, uint8_t stackCount) + :mFullUri(fullUri) + ,mPredict(predict) + ,mTargetURI(targetURI) + ,mSourceURI(sourceURI) + ,mVerifier(verifier) + ,mStackCount(stackCount) + ,mPredictor(predictor) +{ + mStartTime = TimeStamp::Now(); + if (mPredict) { + mPredictReason = reason.mPredict; + } else { + mLearnReason = reason.mLearn; + } +} + +Predictor::Action::~Action() +{ } + +NS_IMETHODIMP +Predictor::Action::OnCacheEntryCheck(nsICacheEntry *entry, + nsIApplicationCache *appCache, + uint32_t *result) +{ + *result = nsICacheEntryOpenCallback::ENTRY_WANTED; + return NS_OK; +} + +NS_IMETHODIMP +Predictor::Action::OnCacheEntryAvailable(nsICacheEntry *entry, bool isNew, + nsIApplicationCache *appCache, + nsresult result) +{ + MOZ_ASSERT(NS_IsMainThread(), "Got cache entry off main thread!"); + + nsAutoCString targetURI, sourceURI; + mTargetURI->GetAsciiSpec(targetURI); + if (mSourceURI) { + mSourceURI->GetAsciiSpec(sourceURI); + } + PREDICTOR_LOG(("OnCacheEntryAvailable %p called. entry=%p mFullUri=%d mPredict=%d " + "mPredictReason=%d mLearnReason=%d mTargetURI=%s " + "mSourceURI=%s mStackCount=%d isNew=%d result=0x%08x", + this, entry, mFullUri, mPredict, mPredictReason, mLearnReason, + targetURI.get(), sourceURI.get(), mStackCount, + isNew, result)); + if (NS_FAILED(result)) { + PREDICTOR_LOG(("OnCacheEntryAvailable %p FAILED to get cache entry (0x%08X). " + "Aborting.", this, result)); + return NS_OK; + } + Telemetry::AccumulateTimeDelta(Telemetry::PREDICTOR_WAIT_TIME, + mStartTime); + if (mPredict) { + bool predicted = mPredictor->PredictInternal(mPredictReason, entry, isNew, + mFullUri, mTargetURI, + mVerifier, mStackCount); + Telemetry::AccumulateTimeDelta( + Telemetry::PREDICTOR_PREDICT_WORK_TIME, mStartTime); + if (predicted) { + Telemetry::AccumulateTimeDelta( + Telemetry::PREDICTOR_PREDICT_TIME_TO_ACTION, mStartTime); + } else { + Telemetry::AccumulateTimeDelta( + Telemetry::PREDICTOR_PREDICT_TIME_TO_INACTION, mStartTime); + } + } else { + mPredictor->LearnInternal(mLearnReason, entry, isNew, mFullUri, mTargetURI, + mSourceURI); + Telemetry::AccumulateTimeDelta( + Telemetry::PREDICTOR_LEARN_WORK_TIME, mStartTime); + } + + return NS_OK; +} + +NS_IMPL_ISUPPORTS(Predictor, + nsINetworkPredictor, + nsIObserver, + nsISpeculativeConnectionOverrider, + nsIInterfaceRequestor, + nsICacheEntryMetaDataVisitor, + nsINetworkPredictorVerifier) + +Predictor::Predictor() + :mInitialized(false) + ,mEnabled(true) + ,mEnableHoverOnSSL(false) + ,mEnablePrefetch(true) + ,mPageDegradationDay(PREDICTOR_PAGE_DELTA_DAY_DEFAULT) + ,mPageDegradationWeek(PREDICTOR_PAGE_DELTA_WEEK_DEFAULT) + ,mPageDegradationMonth(PREDICTOR_PAGE_DELTA_MONTH_DEFAULT) + ,mPageDegradationYear(PREDICTOR_PAGE_DELTA_YEAR_DEFAULT) + ,mPageDegradationMax(PREDICTOR_PAGE_DELTA_MAX_DEFAULT) + ,mSubresourceDegradationDay(PREDICTOR_SUB_DELTA_DAY_DEFAULT) + ,mSubresourceDegradationWeek(PREDICTOR_SUB_DELTA_WEEK_DEFAULT) + ,mSubresourceDegradationMonth(PREDICTOR_SUB_DELTA_MONTH_DEFAULT) + ,mSubresourceDegradationYear(PREDICTOR_SUB_DELTA_YEAR_DEFAULT) + ,mSubresourceDegradationMax(PREDICTOR_SUB_DELTA_MAX_DEFAULT) + ,mPrefetchRollingLoadCount(PREFETCH_ROLLING_LOAD_DEFAULT) + ,mPrefetchMinConfidence(PREFETCH_MIN_DEFAULT) + ,mPreconnectMinConfidence(PRECONNECT_MIN_DEFAULT) + ,mPreresolveMinConfidence(PRERESOLVE_MIN_DEFAULT) + ,mRedirectLikelyConfidence(REDIRECT_LIKELY_DEFAULT) + ,mPrefetchForceValidFor(PREFETCH_FORCE_VALID_DEFAULT) + ,mMaxResourcesPerEntry(PREDICTOR_MAX_RESOURCES_DEFAULT) + ,mStartupCount(1) + ,mMaxURILength(PREDICTOR_MAX_URI_LENGTH_DEFAULT) + ,mDoingTests(false) +{ + MOZ_ASSERT(!sSelf, "multiple Predictor instances!"); + sSelf = this; +} + +Predictor::~Predictor() +{ + if (mInitialized) + Shutdown(); + + sSelf = nullptr; +} + +// Predictor::nsIObserver + +nsresult +Predictor::InstallObserver() +{ + MOZ_ASSERT(NS_IsMainThread(), "Installing observer off main thread"); + + nsresult rv = NS_OK; + nsCOMPtr obs = + mozilla::services::GetObserverService(); + if (!obs) { + return NS_ERROR_NOT_AVAILABLE; + } + + rv = obs->AddObserver(this, NS_XPCOM_SHUTDOWN_OBSERVER_ID, false); + NS_ENSURE_SUCCESS(rv, rv); + + Preferences::AddBoolVarCache(&mEnabled, PREDICTOR_ENABLED_PREF, true); + Preferences::AddBoolVarCache(&mEnableHoverOnSSL, + PREDICTOR_SSL_HOVER_PREF, false); + Preferences::AddBoolVarCache(&mEnablePrefetch, PREDICTOR_PREFETCH_PREF, true); + Preferences::AddIntVarCache(&mPageDegradationDay, + PREDICTOR_PAGE_DELTA_DAY_PREF, + PREDICTOR_PAGE_DELTA_DAY_DEFAULT); + Preferences::AddIntVarCache(&mPageDegradationWeek, + PREDICTOR_PAGE_DELTA_WEEK_PREF, + PREDICTOR_PAGE_DELTA_WEEK_DEFAULT); + Preferences::AddIntVarCache(&mPageDegradationMonth, + PREDICTOR_PAGE_DELTA_MONTH_PREF, + PREDICTOR_PAGE_DELTA_MONTH_DEFAULT); + Preferences::AddIntVarCache(&mPageDegradationYear, + PREDICTOR_PAGE_DELTA_YEAR_PREF, + PREDICTOR_PAGE_DELTA_YEAR_DEFAULT); + Preferences::AddIntVarCache(&mPageDegradationMax, + PREDICTOR_PAGE_DELTA_MAX_PREF, + PREDICTOR_PAGE_DELTA_MAX_DEFAULT); + + Preferences::AddIntVarCache(&mSubresourceDegradationDay, + PREDICTOR_SUB_DELTA_DAY_PREF, + PREDICTOR_SUB_DELTA_DAY_DEFAULT); + Preferences::AddIntVarCache(&mSubresourceDegradationWeek, + PREDICTOR_SUB_DELTA_WEEK_PREF, + PREDICTOR_SUB_DELTA_WEEK_DEFAULT); + Preferences::AddIntVarCache(&mSubresourceDegradationMonth, + PREDICTOR_SUB_DELTA_MONTH_PREF, + PREDICTOR_SUB_DELTA_MONTH_DEFAULT); + Preferences::AddIntVarCache(&mSubresourceDegradationYear, + PREDICTOR_SUB_DELTA_YEAR_PREF, + PREDICTOR_SUB_DELTA_YEAR_DEFAULT); + Preferences::AddIntVarCache(&mSubresourceDegradationMax, + PREDICTOR_SUB_DELTA_MAX_PREF, + PREDICTOR_SUB_DELTA_MAX_DEFAULT); + + Preferences::AddIntVarCache(&mPrefetchRollingLoadCount, + PREDICTOR_PREFETCH_ROLLING_LOAD_PREF, + PREFETCH_ROLLING_LOAD_DEFAULT); + Preferences::AddIntVarCache(&mPrefetchMinConfidence, + PREDICTOR_PREFETCH_MIN_PREF, + PREFETCH_MIN_DEFAULT); + Preferences::AddIntVarCache(&mPreconnectMinConfidence, + PREDICTOR_PRECONNECT_MIN_PREF, + PRECONNECT_MIN_DEFAULT); + Preferences::AddIntVarCache(&mPreresolveMinConfidence, + PREDICTOR_PRERESOLVE_MIN_PREF, + PRERESOLVE_MIN_DEFAULT); + Preferences::AddIntVarCache(&mRedirectLikelyConfidence, + PREDICTOR_REDIRECT_LIKELY_PREF, + REDIRECT_LIKELY_DEFAULT); + + Preferences::AddIntVarCache(&mPrefetchForceValidFor, + PREDICTOR_PREFETCH_FORCE_VALID_PREF, + PREFETCH_FORCE_VALID_DEFAULT); + + Preferences::AddIntVarCache(&mMaxResourcesPerEntry, + PREDICTOR_MAX_RESOURCES_PREF, + PREDICTOR_MAX_RESOURCES_DEFAULT); + + Preferences::AddBoolVarCache(&mCleanedUp, PREDICTOR_CLEANED_UP_PREF, false); + + Preferences::AddUintVarCache(&mMaxURILength, PREDICTOR_MAX_URI_LENGTH_PREF, + PREDICTOR_MAX_URI_LENGTH_DEFAULT); + + Preferences::AddBoolVarCache(&mDoingTests, PREDICTOR_DOING_TESTS_PREF, false); + + if (!mCleanedUp) { + mCleanupTimer = do_CreateInstance("@mozilla.org/timer;1"); + mCleanupTimer->Init(this, 60 * 1000, nsITimer::TYPE_ONE_SHOT); + } + + return rv; +} + +void +Predictor::RemoveObserver() +{ + MOZ_ASSERT(NS_IsMainThread(), "Removing observer off main thread"); + + nsCOMPtr obs = + mozilla::services::GetObserverService(); + if (obs) { + obs->RemoveObserver(this, NS_XPCOM_SHUTDOWN_OBSERVER_ID); + } + + if (mCleanupTimer) { + mCleanupTimer->Cancel(); + mCleanupTimer = nullptr; + } +} + +NS_IMETHODIMP +Predictor::Observe(nsISupports *subject, const char *topic, + const char16_t *data_unicode) +{ + nsresult rv = NS_OK; + MOZ_ASSERT(NS_IsMainThread(), + "Predictor observing something off main thread!"); + + if (!strcmp(NS_XPCOM_SHUTDOWN_OBSERVER_ID, topic)) { + Shutdown(); + } else if (!strcmp("timer-callback", topic)) { + MaybeCleanupOldDBFiles(); + mCleanupTimer = nullptr; + } + + return rv; +} + +// Predictor::nsISpeculativeConnectionOverrider + +NS_IMETHODIMP +Predictor::GetIgnoreIdle(bool *ignoreIdle) +{ + *ignoreIdle = true; + return NS_OK; +} + +NS_IMETHODIMP +Predictor::GetParallelSpeculativeConnectLimit( + uint32_t *parallelSpeculativeConnectLimit) +{ + *parallelSpeculativeConnectLimit = 6; + return NS_OK; +} + +NS_IMETHODIMP +Predictor::GetIsFromPredictor(bool *isFromPredictor) +{ + *isFromPredictor = true; + return NS_OK; +} + +NS_IMETHODIMP +Predictor::GetAllow1918(bool *allow1918) +{ + *allow1918 = false; + return NS_OK; +} + +// Predictor::nsIInterfaceRequestor + +NS_IMETHODIMP +Predictor::GetInterface(const nsIID &iid, void **result) +{ + return QueryInterface(iid, result); +} + +// Predictor::nsICacheEntryMetaDataVisitor + +#define SEEN_META_DATA "predictor::seen" +#define RESOURCE_META_DATA "predictor::resource-count" +#define META_DATA_PREFIX "predictor::" + +static bool +IsURIMetadataElement(const char *key) +{ + return StringBeginsWith(nsDependentCString(key), + NS_LITERAL_CSTRING(META_DATA_PREFIX)) && + !NS_LITERAL_CSTRING(SEEN_META_DATA).Equals(key) && + !NS_LITERAL_CSTRING(RESOURCE_META_DATA).Equals(key); +} + +nsresult +Predictor::OnMetaDataElement(const char *asciiKey, const char *asciiValue) +{ + MOZ_ASSERT(NS_IsMainThread()); + + if (!IsURIMetadataElement(asciiKey)) { + // This isn't a bit of metadata we care about + return NS_OK; + } + + nsCString key, value; + key.AssignASCII(asciiKey); + value.AssignASCII(asciiValue); + mKeysToOperateOn.AppendElement(key); + mValuesToOperateOn.AppendElement(value); + + return NS_OK; +} + +// Predictor::nsINetworkPredictor + +nsresult +Predictor::Init() +{ + MOZ_DIAGNOSTIC_ASSERT(!IsNeckoChild()); + + if (!NS_IsMainThread()) { + MOZ_ASSERT(false, "Predictor::Init called off the main thread!"); + return NS_ERROR_UNEXPECTED; + } + + nsresult rv = NS_OK; + + rv = InstallObserver(); + NS_ENSURE_SUCCESS(rv, rv); + + mLastStartupTime = mStartupTime = NOW_IN_SECONDS(); + + if (!mDNSListener) { + mDNSListener = new DNSListener(); + } + + nsCOMPtr cacheStorageService = + do_GetService("@mozilla.org/netwerk/cache-storage-service;1", &rv); + NS_ENSURE_SUCCESS(rv, rv); + + RefPtr lci = + new LoadContextInfo(false, NeckoOriginAttributes()); + + rv = cacheStorageService->DiskCacheStorage(lci, false, + getter_AddRefs(mCacheDiskStorage)); + NS_ENSURE_SUCCESS(rv, rv); + + mIOService = do_GetService("@mozilla.org/network/io-service;1", &rv); + NS_ENSURE_SUCCESS(rv, rv); + + rv = NS_NewURI(getter_AddRefs(mStartupURI), + "predictor://startup", nullptr, mIOService); + NS_ENSURE_SUCCESS(rv, rv); + + mSpeculativeService = do_QueryInterface(mIOService, &rv); + NS_ENSURE_SUCCESS(rv, rv); + + mDnsService = do_GetService("@mozilla.org/network/dns-service;1", &rv); + NS_ENSURE_SUCCESS(rv, rv); + + mInitialized = true; + + return rv; +} + +namespace { +class PredictorThreadShutdownRunner : public Runnable +{ +public: + PredictorThreadShutdownRunner(nsIThread *ioThread, bool success) + :mIOThread(ioThread) + ,mSuccess(success) + { } + ~PredictorThreadShutdownRunner() { } + + NS_IMETHOD Run() override + { + MOZ_ASSERT(NS_IsMainThread(), "Shutting down io thread off main thread!"); + if (mSuccess) { + // This means the cleanup happened. Mark so we don't try in the + // future. + Preferences::SetBool(PREDICTOR_CLEANED_UP_PREF, true); + } + return mIOThread->AsyncShutdown(); + } + +private: + nsCOMPtr mIOThread; + bool mSuccess; +}; + +class PredictorOldCleanupRunner : public Runnable +{ +public: + PredictorOldCleanupRunner(nsIThread *ioThread, nsIFile *dbFile) + :mIOThread(ioThread) + ,mDBFile(dbFile) + { } + + ~PredictorOldCleanupRunner() { } + + NS_IMETHOD Run() override + { + MOZ_ASSERT(!NS_IsMainThread(), "Cleaning up old files on main thread!"); + nsresult rv = CheckForAndDeleteOldDBFiles(); + RefPtr runner = + new PredictorThreadShutdownRunner(mIOThread, NS_SUCCEEDED(rv)); + NS_DispatchToMainThread(runner); + return NS_OK; + } + +private: + nsresult CheckForAndDeleteOldDBFiles() + { + nsCOMPtr oldDBFile; + nsresult rv = mDBFile->GetParent(getter_AddRefs(oldDBFile)); + NS_ENSURE_SUCCESS(rv, rv); + + rv = oldDBFile->AppendNative(NS_LITERAL_CSTRING("seer.sqlite")); + NS_ENSURE_SUCCESS(rv, rv); + + bool fileExists = false; + rv = oldDBFile->Exists(&fileExists); + NS_ENSURE_SUCCESS(rv, rv); + + if (fileExists) { + rv = oldDBFile->Remove(false); + NS_ENSURE_SUCCESS(rv, rv); + } + + fileExists = false; + rv = mDBFile->Exists(&fileExists); + NS_ENSURE_SUCCESS(rv, rv); + + if (fileExists) { + rv = mDBFile->Remove(false); + } + + return rv; + } + + nsCOMPtr mIOThread; + nsCOMPtr mDBFile; +}; + +} // namespace + +void +Predictor::MaybeCleanupOldDBFiles() +{ + MOZ_ASSERT(NS_IsMainThread()); + + if (!mEnabled || mCleanedUp) { + return; + } + + mCleanedUp = true; + + // This is used for cleaning up junk left over from the old backend + // built on top of sqlite, if necessary. + nsCOMPtr dbFile; + nsresult rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, + getter_AddRefs(dbFile)); + RETURN_IF_FAILED(rv); + rv = dbFile->AppendNative(NS_LITERAL_CSTRING("netpredictions.sqlite")); + RETURN_IF_FAILED(rv); + + nsCOMPtr ioThread; + rv = NS_NewNamedThread("NetPredictClean", getter_AddRefs(ioThread)); + RETURN_IF_FAILED(rv); + + RefPtr runner = + new PredictorOldCleanupRunner(ioThread, dbFile); + ioThread->Dispatch(runner, NS_DISPATCH_NORMAL); +} + +void +Predictor::Shutdown() +{ + if (!NS_IsMainThread()) { + MOZ_ASSERT(false, "Predictor::Shutdown called off the main thread!"); + return; + } + + RemoveObserver(); + + mInitialized = false; +} + +nsresult +Predictor::Create(nsISupports *aOuter, const nsIID& aIID, + void **aResult) +{ + MOZ_ASSERT(NS_IsMainThread()); + + nsresult rv; + + if (aOuter != nullptr) { + return NS_ERROR_NO_AGGREGATION; + } + + RefPtr svc = new Predictor(); + if (IsNeckoChild()) { + // Child threads only need to be call into the public interface methods + // so we don't bother with initialization + return svc->QueryInterface(aIID, aResult); + } + + rv = svc->Init(); + if (NS_FAILED(rv)) { + PREDICTOR_LOG(("Failed to initialize predictor, predictor will be a noop")); + } + + // We treat init failure the same as the service being disabled, since this + // is all an optimization anyway. No need to freak people out. That's why we + // gladly continue on QI'ing here. + rv = svc->QueryInterface(aIID, aResult); + + return rv; +} + +// Called from the main thread to initiate predictive actions +NS_IMETHODIMP +Predictor::Predict(nsIURI *targetURI, nsIURI *sourceURI, + PredictorPredictReason reason, nsILoadContext *loadContext, + nsINetworkPredictorVerifier *verifier) +{ + MOZ_ASSERT(NS_IsMainThread(), + "Predictor interface methods must be called on the main thread"); + + PREDICTOR_LOG(("Predictor::Predict")); + + if (IsNeckoChild()) { + MOZ_DIAGNOSTIC_ASSERT(gNeckoChild); + + PREDICTOR_LOG((" called on child process")); + + ipc::OptionalURIParams serTargetURI, serSourceURI; + SerializeURI(targetURI, serTargetURI); + SerializeURI(sourceURI, serSourceURI); + + IPC::SerializedLoadContext serLoadContext; + serLoadContext.Init(loadContext); + + // If two different threads are predicting concurently, this will be + // overwritten. Thankfully, we only use this in tests, which will + // overwrite mVerifier perhaps multiple times for each individual test; + // however, within each test, the multiple predict calls should have the + // same verifier. + if (verifier) { + PREDICTOR_LOG((" was given a verifier")); + mChildVerifier = verifier; + } + PREDICTOR_LOG((" forwarding to parent process")); + gNeckoChild->SendPredPredict(serTargetURI, serSourceURI, + reason, serLoadContext, verifier); + return NS_OK; + } + + PREDICTOR_LOG((" called on parent process")); + + if (!mInitialized) { + PREDICTOR_LOG((" not initialized")); + return NS_OK; + } + + if (!mEnabled) { + PREDICTOR_LOG((" not enabled")); + return NS_OK; + } + + if (loadContext && loadContext->UsePrivateBrowsing()) { + // Don't want to do anything in PB mode + PREDICTOR_LOG((" in PB mode")); + return NS_OK; + } + + if (!IsNullOrHttp(targetURI) || !IsNullOrHttp(sourceURI)) { + // Nothing we can do for non-HTTP[S] schemes + PREDICTOR_LOG((" got non-http[s] URI")); + return NS_OK; + } + + // Ensure we've been given the appropriate arguments for the kind of + // prediction we're being asked to do + nsCOMPtr uriKey = targetURI; + nsCOMPtr originKey; + switch (reason) { + case nsINetworkPredictor::PREDICT_LINK: + if (!targetURI || !sourceURI) { + PREDICTOR_LOG((" link invalid URI state")); + return NS_ERROR_INVALID_ARG; + } + // Link hover is a special case where we can predict without hitting the + // db, so let's go ahead and fire off that prediction here. + PredictForLink(targetURI, sourceURI, verifier); + return NS_OK; + case nsINetworkPredictor::PREDICT_LOAD: + if (!targetURI || sourceURI) { + PREDICTOR_LOG((" load invalid URI state")); + return NS_ERROR_INVALID_ARG; + } + break; + case nsINetworkPredictor::PREDICT_STARTUP: + if (targetURI || sourceURI) { + PREDICTOR_LOG((" startup invalid URI state")); + return NS_ERROR_INVALID_ARG; + } + uriKey = mStartupURI; + originKey = mStartupURI; + break; + default: + PREDICTOR_LOG((" invalid reason")); + return NS_ERROR_INVALID_ARG; + } + + Predictor::Reason argReason; + argReason.mPredict = reason; + + // First we open the regular cache entry, to ensure we don't gum up the works + // waiting on the less-important predictor-only cache entry + RefPtr uriAction = + new Predictor::Action(Predictor::Action::IS_FULL_URI, + Predictor::Action::DO_PREDICT, argReason, targetURI, + nullptr, verifier, this); + nsAutoCString uriKeyStr; + uriKey->GetAsciiSpec(uriKeyStr); + PREDICTOR_LOG((" Predict uri=%s reason=%d action=%p", uriKeyStr.get(), + reason, uriAction.get())); + uint32_t openFlags = nsICacheStorage::OPEN_READONLY | + nsICacheStorage::OPEN_SECRETLY | + nsICacheStorage::OPEN_PRIORITY | + nsICacheStorage::CHECK_MULTITHREADED; + mCacheDiskStorage->AsyncOpenURI(uriKey, EmptyCString(), openFlags, uriAction); + + // Now we do the origin-only (and therefore predictor-only) entry + nsCOMPtr targetOrigin; + nsresult rv = ExtractOrigin(uriKey, getter_AddRefs(targetOrigin), mIOService); + NS_ENSURE_SUCCESS(rv, rv); + if (!originKey) { + originKey = targetOrigin; + } + + RefPtr originAction = + new Predictor::Action(Predictor::Action::IS_ORIGIN, + Predictor::Action::DO_PREDICT, argReason, + targetOrigin, nullptr, verifier, this); + nsAutoCString originKeyStr; + originKey->GetAsciiSpec(originKeyStr); + PREDICTOR_LOG((" Predict origin=%s reason=%d action=%p", originKeyStr.get(), + reason, originAction.get())); + openFlags = nsICacheStorage::OPEN_READONLY | + nsICacheStorage::OPEN_SECRETLY | + nsICacheStorage::CHECK_MULTITHREADED; + mCacheDiskStorage->AsyncOpenURI(originKey, + NS_LITERAL_CSTRING(PREDICTOR_ORIGIN_EXTENSION), + openFlags, originAction); + + PREDICTOR_LOG((" predict returning")); + return NS_OK; +} + +bool +Predictor::PredictInternal(PredictorPredictReason reason, nsICacheEntry *entry, + bool isNew, bool fullUri, nsIURI *targetURI, + nsINetworkPredictorVerifier *verifier, + uint8_t stackCount) +{ + MOZ_ASSERT(NS_IsMainThread()); + + PREDICTOR_LOG(("Predictor::PredictInternal")); + bool rv = false; + + if (reason == nsINetworkPredictor::PREDICT_LOAD) { + MaybeLearnForStartup(targetURI, fullUri); + } + + if (isNew) { + // nothing else we can do here + PREDICTOR_LOG((" new entry")); + return rv; + } + + switch (reason) { + case nsINetworkPredictor::PREDICT_LOAD: + rv = PredictForPageload(entry, targetURI, stackCount, fullUri, verifier); + break; + case nsINetworkPredictor::PREDICT_STARTUP: + rv = PredictForStartup(entry, fullUri, verifier); + break; + default: + PREDICTOR_LOG((" invalid reason")); + MOZ_ASSERT(false, "Got unexpected value for prediction reason"); + } + + return rv; +} + +void +Predictor::PredictForLink(nsIURI *targetURI, nsIURI *sourceURI, + nsINetworkPredictorVerifier *verifier) +{ + MOZ_ASSERT(NS_IsMainThread()); + + PREDICTOR_LOG(("Predictor::PredictForLink")); + if (!mSpeculativeService) { + PREDICTOR_LOG((" missing speculative service")); + return; + } + + if (!mEnableHoverOnSSL) { + bool isSSL = false; + sourceURI->SchemeIs("https", &isSSL); + if (isSSL) { + // We don't want to predict from an HTTPS page, to avoid info leakage + PREDICTOR_LOG((" Not predicting for link hover - on an SSL page")); + return; + } + } + + mSpeculativeService->SpeculativeConnect2(targetURI, nullptr, nullptr); + if (verifier) { + PREDICTOR_LOG((" sending verification")); + verifier->OnPredictPreconnect(targetURI); + } +} + +// This is the driver for prediction based on a new pageload. +static const uint8_t MAX_PAGELOAD_DEPTH = 10; +bool +Predictor::PredictForPageload(nsICacheEntry *entry, nsIURI *targetURI, + uint8_t stackCount, bool fullUri, + nsINetworkPredictorVerifier *verifier) +{ + MOZ_ASSERT(NS_IsMainThread()); + + PREDICTOR_LOG(("Predictor::PredictForPageload")); + + if (stackCount > MAX_PAGELOAD_DEPTH) { + PREDICTOR_LOG((" exceeded recursion depth!")); + return false; + } + + uint32_t lastLoad; + nsresult rv = entry->GetLastFetched(&lastLoad); + NS_ENSURE_SUCCESS(rv, false); + + int32_t globalDegradation = CalculateGlobalDegradation(lastLoad); + PREDICTOR_LOG((" globalDegradation = %d", globalDegradation)); + + int32_t loadCount; + rv = entry->GetFetchCount(&loadCount); + NS_ENSURE_SUCCESS(rv, false); + + nsCOMPtr redirectURI; + if (WouldRedirect(entry, loadCount, lastLoad, globalDegradation, + getter_AddRefs(redirectURI))) { + mPreconnects.AppendElement(redirectURI); + Predictor::Reason reason; + reason.mPredict = nsINetworkPredictor::PREDICT_LOAD; + RefPtr redirectAction = + new Predictor::Action(Predictor::Action::IS_FULL_URI, + Predictor::Action::DO_PREDICT, reason, redirectURI, + nullptr, verifier, this, stackCount + 1); + nsAutoCString redirectUriString; + redirectURI->GetAsciiSpec(redirectUriString); + PREDICTOR_LOG((" Predict redirect uri=%s action=%p", redirectUriString.get(), + redirectAction.get())); + uint32_t openFlags = nsICacheStorage::OPEN_READONLY | + nsICacheStorage::OPEN_SECRETLY | + nsICacheStorage::OPEN_PRIORITY | + nsICacheStorage::CHECK_MULTITHREADED; + mCacheDiskStorage->AsyncOpenURI(redirectURI, EmptyCString(), openFlags, + redirectAction); + return RunPredictions(nullptr, verifier); + } + + CalculatePredictions(entry, targetURI, lastLoad, loadCount, globalDegradation, fullUri); + + return RunPredictions(targetURI, verifier); +} + +// This is the driver for predicting at browser startup time based on pages that +// have previously been loaded close to startup. +bool +Predictor::PredictForStartup(nsICacheEntry *entry, bool fullUri, + nsINetworkPredictorVerifier *verifier) +{ + MOZ_ASSERT(NS_IsMainThread()); + + PREDICTOR_LOG(("Predictor::PredictForStartup")); + int32_t globalDegradation = CalculateGlobalDegradation(mLastStartupTime); + CalculatePredictions(entry, nullptr, mLastStartupTime, mStartupCount, + globalDegradation, fullUri); + return RunPredictions(nullptr, verifier); +} + +// This calculates how much to degrade our confidence in our data based on +// the last time this top-level resource was loaded. This "global degradation" +// applies to *all* subresources we have associated with the top-level +// resource. This will be in addition to any reduction in confidence we have +// associated with a particular subresource. +int32_t +Predictor::CalculateGlobalDegradation(uint32_t lastLoad) +{ + MOZ_ASSERT(NS_IsMainThread()); + + int32_t globalDegradation; + uint32_t delta = NOW_IN_SECONDS() - lastLoad; + if (delta < ONE_DAY) { + globalDegradation = mPageDegradationDay; + } else if (delta < ONE_WEEK) { + globalDegradation = mPageDegradationWeek; + } else if (delta < ONE_MONTH) { + globalDegradation = mPageDegradationMonth; + } else if (delta < ONE_YEAR) { + globalDegradation = mPageDegradationYear; + } else { + globalDegradation = mPageDegradationMax; + } + + Telemetry::Accumulate(Telemetry::PREDICTOR_GLOBAL_DEGRADATION, + globalDegradation); + return globalDegradation; +} + +// This calculates our overall confidence that a particular subresource will be +// loaded as part of a top-level load. +// @param hitCount - the number of times we have loaded this subresource as part +// of this top-level load +// @param hitsPossible - the number of times we have performed this top-level +// load +// @param lastHit - the timestamp of the last time we loaded this subresource as +// part of this top-level load +// @param lastPossible - the timestamp of the last time we performed this +// top-level load +// @param globalDegradation - the degradation for this top-level load as +// determined by CalculateGlobalDegradation +int32_t +Predictor::CalculateConfidence(uint32_t hitCount, uint32_t hitsPossible, + uint32_t lastHit, uint32_t lastPossible, + int32_t globalDegradation) +{ + MOZ_ASSERT(NS_IsMainThread()); + + Telemetry::AutoCounter predictionsCalculated; + ++predictionsCalculated; + + if (!hitsPossible) { + return 0; + } + + int32_t baseConfidence = (hitCount * 100) / hitsPossible; + int32_t maxConfidence = 100; + int32_t confidenceDegradation = 0; + + if (lastHit < lastPossible) { + // We didn't load this subresource the last time this top-level load was + // performed, so let's not bother preconnecting (at the very least). + maxConfidence = mPreconnectMinConfidence - 1; + + // Now calculate how much we want to degrade our confidence based on how + // long it's been between the last time we did this top-level load and the + // last time this top-level load included this subresource. + PRTime delta = lastPossible - lastHit; + if (delta == 0) { + confidenceDegradation = 0; + } else if (delta < ONE_DAY) { + confidenceDegradation = mSubresourceDegradationDay; + } else if (delta < ONE_WEEK) { + confidenceDegradation = mSubresourceDegradationWeek; + } else if (delta < ONE_MONTH) { + confidenceDegradation = mSubresourceDegradationMonth; + } else if (delta < ONE_YEAR) { + confidenceDegradation = mSubresourceDegradationYear; + } else { + confidenceDegradation = mSubresourceDegradationMax; + maxConfidence = 0; + } + } + + // Calculate our confidence and clamp it to between 0 and maxConfidence + // (<= 100) + int32_t confidence = baseConfidence - confidenceDegradation - globalDegradation; + confidence = std::max(confidence, 0); + confidence = std::min(confidence, maxConfidence); + + Telemetry::Accumulate(Telemetry::PREDICTOR_BASE_CONFIDENCE, baseConfidence); + Telemetry::Accumulate(Telemetry::PREDICTOR_SUBRESOURCE_DEGRADATION, + confidenceDegradation); + Telemetry::Accumulate(Telemetry::PREDICTOR_CONFIDENCE, confidence); + return confidence; +} + +static void +MakeMetadataEntry(const uint32_t hitCount, const uint32_t lastHit, + const uint32_t flags, nsCString &newValue) +{ + newValue.Truncate(); + newValue.AppendInt(METADATA_VERSION); + newValue.Append(','); + newValue.AppendInt(hitCount); + newValue.Append(','); + newValue.AppendInt(lastHit); + newValue.Append(','); + newValue.AppendInt(flags); +} + +// On every page load, the rolling window gets shifted by one bit, leaving the +// lowest bit at 0, to indicate that the subresource in question has not been +// seen on the most recent page load. If, at some point later during the page load, +// the subresource is seen again, we will then set the lowest bit to 1. This is +// how we keep track of how many of the last n pageloads (for n <= 20) a particular +// subresource has been seen. +// The rolling window is kept in the upper 20 bits of the flags element of the +// metadata. This saves 12 bits for regular old flags. +void +Predictor::UpdateRollingLoadCount(nsICacheEntry *entry, const uint32_t flags, + const char *key, const uint32_t hitCount, + const uint32_t lastHit) +{ + // Extract just the rolling load count from the flags, shift it to clear the + // lowest bit, and put the new value with the existing flags. + uint32_t rollingLoadCount = flags & ~kFlagsMask; + rollingLoadCount <<= 1; + uint32_t newFlags = (flags & kFlagsMask) | rollingLoadCount; + + // Finally, update the metadata on the cache entry. + nsAutoCString newValue; + MakeMetadataEntry(hitCount, lastHit, newFlags, newValue); + entry->SetMetaDataElement(key, newValue.BeginReading()); +} + +void +Predictor::SanitizePrefs() +{ + if (mPrefetchRollingLoadCount < 0) { + mPrefetchRollingLoadCount = 0; + } else if (mPrefetchRollingLoadCount > kMaxPrefetchRollingLoadCount) { + mPrefetchRollingLoadCount = kMaxPrefetchRollingLoadCount; + } +} + +void +Predictor::CalculatePredictions(nsICacheEntry *entry, nsIURI *referrer, + uint32_t lastLoad, uint32_t loadCount, + int32_t globalDegradation, bool fullUri) +{ + MOZ_ASSERT(NS_IsMainThread()); + + SanitizePrefs(); + + // Since the visitor gets called under a cache lock, all we do there is get + // copies of the keys/values we care about, and then do the real work here + entry->VisitMetaData(this); + nsTArray keysToOperateOn, valuesToOperateOn; + keysToOperateOn.SwapElements(mKeysToOperateOn); + valuesToOperateOn.SwapElements(mValuesToOperateOn); + + MOZ_ASSERT(keysToOperateOn.Length() == valuesToOperateOn.Length()); + for (size_t i = 0; i < keysToOperateOn.Length(); ++i) { + const char *key = keysToOperateOn[i].BeginReading(); + const char *value = valuesToOperateOn[i].BeginReading(); + + nsCOMPtr uri; + uint32_t hitCount, lastHit, flags; + if (!ParseMetaDataEntry(key, value, getter_AddRefs(uri), hitCount, lastHit, flags)) { + // This failed, get rid of it so we don't waste space + entry->SetMetaDataElement(key, nullptr); + continue; + } + + int32_t confidence = CalculateConfidence(hitCount, loadCount, lastHit, + lastLoad, globalDegradation); + if (fullUri) { + UpdateRollingLoadCount(entry, flags, key, hitCount, lastHit); + } + PREDICTOR_LOG(("CalculatePredictions key=%s value=%s confidence=%d", key, value, confidence)); + if (!fullUri) { + // Not full URI - don't prefetch! No sense in it! + PREDICTOR_LOG((" forcing non-cacheability - not full URI")); + flags &= ~FLAG_PREFETCHABLE; + } else if (!referrer) { + // No referrer means we can't prefetch, so pretend it's non-cacheable, + // no matter what. + PREDICTOR_LOG((" forcing non-cacheability - no referrer")); + flags &= ~FLAG_PREFETCHABLE; + } else { + uint32_t expectedRollingLoadCount = (1 << mPrefetchRollingLoadCount) - 1; + expectedRollingLoadCount <<= kRollingLoadOffset; + if ((flags & expectedRollingLoadCount) != expectedRollingLoadCount) { + PREDICTOR_LOG((" forcing non-cacheability - missed a load")); + flags &= ~FLAG_PREFETCHABLE; + } + } + + PREDICTOR_LOG((" setting up prediction")); + SetupPrediction(confidence, flags, uri); + } +} + +// (Maybe) adds a predictive action to the prediction runner, based on our +// calculated confidence for the subresource in question. +void +Predictor::SetupPrediction(int32_t confidence, uint32_t flags, nsIURI *uri) +{ + MOZ_ASSERT(NS_IsMainThread()); + + nsAutoCString uriStr; + uri->GetAsciiSpec(uriStr); + PREDICTOR_LOG(("SetupPrediction mEnablePrefetch=%d mPrefetchMinConfidence=%d " + "mPreconnectMinConfidence=%d mPreresolveMinConfidence=%d " + "flags=%d confidence=%d uri=%s", mEnablePrefetch, + mPrefetchMinConfidence, mPreconnectMinConfidence, + mPreresolveMinConfidence, flags, confidence, uriStr.get())); + if (mEnablePrefetch && (flags & FLAG_PREFETCHABLE) && + (mPrefetchRollingLoadCount || (confidence >= mPrefetchMinConfidence))) { + mPrefetches.AppendElement(uri); + } else if (confidence >= mPreconnectMinConfidence) { + mPreconnects.AppendElement(uri); + } else if (confidence >= mPreresolveMinConfidence) { + mPreresolves.AppendElement(uri); + } +} + +nsresult +Predictor::Prefetch(nsIURI *uri, nsIURI *referrer, + nsINetworkPredictorVerifier *verifier) +{ + nsAutoCString strUri, strReferrer; + uri->GetAsciiSpec(strUri); + referrer->GetAsciiSpec(strReferrer); + PREDICTOR_LOG(("Predictor::Prefetch uri=%s referrer=%s verifier=%p", + strUri.get(), strReferrer.get(), verifier)); + nsCOMPtr channel; + nsresult rv = NS_NewChannel(getter_AddRefs(channel), uri, + nsContentUtils::GetSystemPrincipal(), + nsILoadInfo::SEC_ALLOW_CROSS_ORIGIN_DATA_IS_NULL, + nsIContentPolicy::TYPE_OTHER, + nullptr, /* aLoadGroup */ + nullptr, /* aCallbacks */ + nsIRequest::LOAD_BACKGROUND); + + if (NS_FAILED(rv)) { + PREDICTOR_LOG((" NS_NewChannel failed rv=0x%X", rv)); + return rv; + } + + nsCOMPtr httpChannel; + httpChannel = do_QueryInterface(channel); + if (!httpChannel) { + PREDICTOR_LOG((" Could not get HTTP Channel from new channel!")); + return NS_ERROR_UNEXPECTED; + } + + httpChannel->SetReferrer(referrer); + // XXX - set a header here to indicate this is a prefetch? + + nsCOMPtr listener = new PrefetchListener(verifier, uri, + this); + PREDICTOR_LOG((" calling AsyncOpen2 listener=%p channel=%p", listener.get(), + channel.get())); + rv = channel->AsyncOpen2(listener); + if (NS_FAILED(rv)) { + PREDICTOR_LOG((" AsyncOpen2 failed rv=0x%X", rv)); + } + + return rv; +} + +// Runs predictions that have been set up. +bool +Predictor::RunPredictions(nsIURI *referrer, nsINetworkPredictorVerifier *verifier) +{ + MOZ_ASSERT(NS_IsMainThread(), "Running prediction off main thread"); + + PREDICTOR_LOG(("Predictor::RunPredictions")); + + bool predicted = false; + uint32_t len, i; + + nsTArray> prefetches, preconnects, preresolves; + prefetches.SwapElements(mPrefetches); + preconnects.SwapElements(mPreconnects); + preresolves.SwapElements(mPreresolves); + + Telemetry::AutoCounter totalPredictions; + Telemetry::AutoCounter totalPrefetches; + Telemetry::AutoCounter totalPreconnects; + Telemetry::AutoCounter totalPreresolves; + + len = prefetches.Length(); + for (i = 0; i < len; ++i) { + PREDICTOR_LOG((" doing prefetch")); + nsCOMPtr uri = prefetches[i]; + if (NS_SUCCEEDED(Prefetch(uri, referrer, verifier))) { + ++totalPredictions; + ++totalPrefetches; + predicted = true; + } + } + + len = preconnects.Length(); + for (i = 0; i < len; ++i) { + PREDICTOR_LOG((" doing preconnect")); + nsCOMPtr uri = preconnects[i]; + ++totalPredictions; + ++totalPreconnects; + mSpeculativeService->SpeculativeConnect2(uri, nullptr, this); + predicted = true; + if (verifier) { + PREDICTOR_LOG((" sending preconnect verification")); + verifier->OnPredictPreconnect(uri); + } + } + + len = preresolves.Length(); + nsCOMPtr mainThread = do_GetMainThread(); + for (i = 0; i < len; ++i) { + nsCOMPtr uri = preresolves[i]; + ++totalPredictions; + ++totalPreresolves; + nsAutoCString hostname; + uri->GetAsciiHost(hostname); + PREDICTOR_LOG((" doing preresolve %s", hostname.get())); + nsCOMPtr tmpCancelable; + mDnsService->AsyncResolve(hostname, + (nsIDNSService::RESOLVE_PRIORITY_MEDIUM | + nsIDNSService::RESOLVE_SPECULATE), + mDNSListener, nullptr, + getter_AddRefs(tmpCancelable)); + predicted = true; + if (verifier) { + PREDICTOR_LOG((" sending preresolve verification")); + verifier->OnPredictDNS(uri); + } + } + + return predicted; +} + +// Find out if a top-level page is likely to redirect. +bool +Predictor::WouldRedirect(nsICacheEntry *entry, uint32_t loadCount, + uint32_t lastLoad, int32_t globalDegradation, + nsIURI **redirectURI) +{ + // TODO - not doing redirects for first go around + MOZ_ASSERT(NS_IsMainThread()); + + return false; +} + +// Called from the main thread to update the database +NS_IMETHODIMP +Predictor::Learn(nsIURI *targetURI, nsIURI *sourceURI, + PredictorLearnReason reason, + nsILoadContext *loadContext) +{ + MOZ_ASSERT(NS_IsMainThread(), + "Predictor interface methods must be called on the main thread"); + + PREDICTOR_LOG(("Predictor::Learn")); + + if (IsNeckoChild()) { + MOZ_DIAGNOSTIC_ASSERT(gNeckoChild); + + PREDICTOR_LOG((" called on child process")); + + ipc::URIParams serTargetURI; + SerializeURI(targetURI, serTargetURI); + + ipc::OptionalURIParams serSourceURI; + SerializeURI(sourceURI, serSourceURI); + + IPC::SerializedLoadContext serLoadContext; + serLoadContext.Init(loadContext); + + PREDICTOR_LOG((" forwarding to parent")); + gNeckoChild->SendPredLearn(serTargetURI, serSourceURI, reason, + serLoadContext); + return NS_OK; + } + + PREDICTOR_LOG((" called on parent process")); + + if (!mInitialized) { + PREDICTOR_LOG((" not initialized")); + return NS_OK; + } + + if (!mEnabled) { + PREDICTOR_LOG((" not enabled")); + return NS_OK; + } + + if (loadContext && loadContext->UsePrivateBrowsing()) { + // Don't want to do anything in PB mode + PREDICTOR_LOG((" in PB mode")); + return NS_OK; + } + + if (!IsNullOrHttp(targetURI) || !IsNullOrHttp(sourceURI)) { + PREDICTOR_LOG((" got non-HTTP[S] URI")); + return NS_ERROR_INVALID_ARG; + } + + nsCOMPtr targetOrigin; + nsCOMPtr sourceOrigin; + nsCOMPtr uriKey; + nsCOMPtr originKey; + nsresult rv; + + switch (reason) { + case nsINetworkPredictor::LEARN_LOAD_TOPLEVEL: + if (!targetURI || sourceURI) { + PREDICTOR_LOG((" load toplevel invalid URI state")); + return NS_ERROR_INVALID_ARG; + } + rv = ExtractOrigin(targetURI, getter_AddRefs(targetOrigin), mIOService); + NS_ENSURE_SUCCESS(rv, rv); + uriKey = targetURI; + originKey = targetOrigin; + break; + case nsINetworkPredictor::LEARN_STARTUP: + if (!targetURI || sourceURI) { + PREDICTOR_LOG((" startup invalid URI state")); + return NS_ERROR_INVALID_ARG; + } + rv = ExtractOrigin(targetURI, getter_AddRefs(targetOrigin), mIOService); + NS_ENSURE_SUCCESS(rv, rv); + uriKey = mStartupURI; + originKey = mStartupURI; + break; + case nsINetworkPredictor::LEARN_LOAD_REDIRECT: + case nsINetworkPredictor::LEARN_LOAD_SUBRESOURCE: + if (!targetURI || !sourceURI) { + PREDICTOR_LOG((" redirect/subresource invalid URI state")); + return NS_ERROR_INVALID_ARG; + } + rv = ExtractOrigin(targetURI, getter_AddRefs(targetOrigin), mIOService); + NS_ENSURE_SUCCESS(rv, rv); + rv = ExtractOrigin(sourceURI, getter_AddRefs(sourceOrigin), mIOService); + NS_ENSURE_SUCCESS(rv, rv); + uriKey = sourceURI; + originKey = sourceOrigin; + break; + default: + PREDICTOR_LOG((" invalid reason")); + return NS_ERROR_INVALID_ARG; + } + + Telemetry::AutoCounter learnAttempts; + ++learnAttempts; + + Predictor::Reason argReason; + argReason.mLearn = reason; + + // We always open the full uri (general cache) entry first, so we don't gum up + // the works waiting on predictor-only entries to open + RefPtr uriAction = + new Predictor::Action(Predictor::Action::IS_FULL_URI, + Predictor::Action::DO_LEARN, argReason, targetURI, + sourceURI, nullptr, this); + nsAutoCString uriKeyStr, targetUriStr, sourceUriStr; + uriKey->GetAsciiSpec(uriKeyStr); + targetURI->GetAsciiSpec(targetUriStr); + if (sourceURI) { + sourceURI->GetAsciiSpec(sourceUriStr); + } + PREDICTOR_LOG((" Learn uriKey=%s targetURI=%s sourceURI=%s reason=%d " + "action=%p", uriKeyStr.get(), targetUriStr.get(), + sourceUriStr.get(), reason, uriAction.get())); + // For learning full URI things, we *always* open readonly and secretly, as we + // rely on actual pageloads to update the entry's metadata for us. + uint32_t uriOpenFlags = nsICacheStorage::OPEN_READONLY | + nsICacheStorage::OPEN_SECRETLY | + nsICacheStorage::CHECK_MULTITHREADED; + if (reason == nsINetworkPredictor::LEARN_LOAD_TOPLEVEL) { + // Learning for toplevel we want to open the full uri entry priority, since + // it's likely this entry will be used soon anyway, and we want this to be + // opened ASAP. + uriOpenFlags |= nsICacheStorage::OPEN_PRIORITY; + } + mCacheDiskStorage->AsyncOpenURI(uriKey, EmptyCString(), uriOpenFlags, + uriAction); + + // Now we open the origin-only (and therefore predictor-only) entry + RefPtr originAction = + new Predictor::Action(Predictor::Action::IS_ORIGIN, + Predictor::Action::DO_LEARN, argReason, targetOrigin, + sourceOrigin, nullptr, this); + nsAutoCString originKeyStr, targetOriginStr, sourceOriginStr; + originKey->GetAsciiSpec(originKeyStr); + targetOrigin->GetAsciiSpec(targetOriginStr); + if (sourceOrigin) { + sourceOrigin->GetAsciiSpec(sourceOriginStr); + } + PREDICTOR_LOG((" Learn originKey=%s targetOrigin=%s sourceOrigin=%s reason=%d " + "action=%p", originKeyStr.get(), targetOriginStr.get(), + sourceOriginStr.get(), reason, originAction.get())); + uint32_t originOpenFlags; + if (reason == nsINetworkPredictor::LEARN_LOAD_TOPLEVEL) { + // This is the only case when we want to update the 'last used' metadata on + // the cache entry we're getting. This only applies to predictor-specific + // entries. + originOpenFlags = nsICacheStorage::OPEN_NORMALLY | + nsICacheStorage::CHECK_MULTITHREADED; + } else { + originOpenFlags = nsICacheStorage::OPEN_READONLY | + nsICacheStorage::OPEN_SECRETLY | + nsICacheStorage::CHECK_MULTITHREADED; + } + mCacheDiskStorage->AsyncOpenURI(originKey, + NS_LITERAL_CSTRING(PREDICTOR_ORIGIN_EXTENSION), + originOpenFlags, originAction); + + PREDICTOR_LOG(("Predictor::Learn returning")); + return NS_OK; +} + +void +Predictor::LearnInternal(PredictorLearnReason reason, nsICacheEntry *entry, + bool isNew, bool fullUri, nsIURI *targetURI, + nsIURI *sourceURI) +{ + MOZ_ASSERT(NS_IsMainThread()); + + PREDICTOR_LOG(("Predictor::LearnInternal")); + + nsCString junk; + if (!fullUri && reason == nsINetworkPredictor::LEARN_LOAD_TOPLEVEL && + NS_FAILED(entry->GetMetaDataElement(SEEN_META_DATA, getter_Copies(junk)))) { + // This is an origin-only entry that we haven't seen before. Let's mark it + // as seen. + PREDICTOR_LOG((" marking new origin entry as seen")); + nsresult rv = entry->SetMetaDataElement(SEEN_META_DATA, "1"); + if (NS_FAILED(rv)) { + PREDICTOR_LOG((" failed to mark origin entry seen")); + return; + } + + // Need to ensure someone else can get to the entry if necessary + entry->MetaDataReady(); + return; + } + + switch (reason) { + case nsINetworkPredictor::LEARN_LOAD_TOPLEVEL: + // This case only exists to be used during tests - code outside the + // predictor tests should NEVER call Learn with LEARN_LOAD_TOPLEVEL. + // The predictor xpcshell test needs this branch, however, because we + // have no real page loads in xpcshell, and this is how we fake it up + // so that all the work that normally happens behind the scenes in a + // page load can be done for testing purposes. + if (fullUri && mDoingTests) { + PREDICTOR_LOG((" WARNING - updating rolling load count. " + "If you see this outside tests, you did it wrong")); + SanitizePrefs(); + + // Since the visitor gets called under a cache lock, all we do there is get + // copies of the keys/values we care about, and then do the real work here + entry->VisitMetaData(this); + nsTArray keysToOperateOn, valuesToOperateOn; + keysToOperateOn.SwapElements(mKeysToOperateOn); + valuesToOperateOn.SwapElements(mValuesToOperateOn); + + MOZ_ASSERT(keysToOperateOn.Length() == valuesToOperateOn.Length()); + for (size_t i = 0; i < keysToOperateOn.Length(); ++i) { + const char *key = keysToOperateOn[i].BeginReading(); + const char *value = valuesToOperateOn[i].BeginReading(); + + nsCOMPtr uri; + uint32_t hitCount, lastHit, flags; + if (!ParseMetaDataEntry(nullptr, value, nullptr, hitCount, lastHit, flags)) { + // This failed, get rid of it so we don't waste space + entry->SetMetaDataElement(key, nullptr); + continue; + } + UpdateRollingLoadCount(entry, flags, key, hitCount, lastHit); + } + } else { + PREDICTOR_LOG((" nothing to do for toplevel")); + } + break; + case nsINetworkPredictor::LEARN_LOAD_REDIRECT: + if (fullUri) { + LearnForRedirect(entry, targetURI); + } + break; + case nsINetworkPredictor::LEARN_LOAD_SUBRESOURCE: + LearnForSubresource(entry, targetURI); + break; + case nsINetworkPredictor::LEARN_STARTUP: + LearnForStartup(entry, targetURI); + break; + default: + PREDICTOR_LOG((" unexpected reason value")); + MOZ_ASSERT(false, "Got unexpected value for learn reason!"); + } +} + +NS_IMPL_ISUPPORTS(Predictor::SpaceCleaner, nsICacheEntryMetaDataVisitor) + +NS_IMETHODIMP +Predictor::SpaceCleaner::OnMetaDataElement(const char *key, const char *value) +{ + MOZ_ASSERT(NS_IsMainThread()); + + if (!IsURIMetadataElement(key)) { + // This isn't a bit of metadata we care about + return NS_OK; + } + + uint32_t hitCount, lastHit, flags; + bool ok = mPredictor->ParseMetaDataEntry(nullptr, value, nullptr, + hitCount, lastHit, flags); + + if (!ok) { + // Couldn't parse this one, just get rid of it + nsCString nsKey; + nsKey.AssignASCII(key); + mLongKeysToDelete.AppendElement(nsKey); + return NS_OK; + } + + nsCString uri(key + (sizeof(META_DATA_PREFIX) - 1)); + uint32_t uriLength = uri.Length(); + if (uriLength > mPredictor->mMaxURILength) { + // Default to getting rid of URIs that are too long and were put in before + // we had our limit on URI length, in order to free up some space. + nsCString nsKey; + nsKey.AssignASCII(key); + mLongKeysToDelete.AppendElement(nsKey); + return NS_OK; + } + + if (!mLRUKeyToDelete || lastHit < mLRUStamp) { + mLRUKeyToDelete = key; + mLRUStamp = lastHit; + } + + return NS_OK; +} + +void +Predictor::SpaceCleaner::Finalize(nsICacheEntry *entry) +{ + MOZ_ASSERT(NS_IsMainThread()); + + if (mLRUKeyToDelete) { + entry->SetMetaDataElement(mLRUKeyToDelete, nullptr); + } + + for (size_t i = 0; i < mLongKeysToDelete.Length(); ++i) { + entry->SetMetaDataElement(mLongKeysToDelete[i].BeginReading(), nullptr); + } +} + +// Called when a subresource has been hit from a top-level load. Uses the two +// helper functions above to update the database appropriately. +void +Predictor::LearnForSubresource(nsICacheEntry *entry, nsIURI *targetURI) +{ + MOZ_ASSERT(NS_IsMainThread()); + + PREDICTOR_LOG(("Predictor::LearnForSubresource")); + + uint32_t lastLoad; + nsresult rv = entry->GetLastFetched(&lastLoad); + RETURN_IF_FAILED(rv); + + int32_t loadCount; + rv = entry->GetFetchCount(&loadCount); + RETURN_IF_FAILED(rv); + + nsCString key; + key.AssignLiteral(META_DATA_PREFIX); + nsCString uri; + targetURI->GetAsciiSpec(uri); + key.Append(uri); + if (uri.Length() > mMaxURILength) { + // We do this to conserve space/prevent OOMs + PREDICTOR_LOG((" uri too long!")); + entry->SetMetaDataElement(key.BeginReading(), nullptr); + return; + } + + nsCString value; + rv = entry->GetMetaDataElement(key.BeginReading(), getter_Copies(value)); + + uint32_t hitCount, lastHit, flags; + bool isNewResource = (NS_FAILED(rv) || + !ParseMetaDataEntry(nullptr, value.BeginReading(), + nullptr, hitCount, lastHit, flags)); + + int32_t resourceCount = 0; + if (isNewResource) { + // This is a new addition + PREDICTOR_LOG((" new resource")); + nsCString s; + rv = entry->GetMetaDataElement(RESOURCE_META_DATA, getter_Copies(s)); + if (NS_SUCCEEDED(rv)) { + resourceCount = atoi(s.BeginReading()); + } + if (resourceCount >= mMaxResourcesPerEntry) { + RefPtr cleaner = + new Predictor::SpaceCleaner(this); + entry->VisitMetaData(cleaner); + cleaner->Finalize(entry); + } else { + ++resourceCount; + } + nsAutoCString count; + count.AppendInt(resourceCount); + rv = entry->SetMetaDataElement(RESOURCE_META_DATA, count.BeginReading()); + if (NS_FAILED(rv)) { + PREDICTOR_LOG((" failed to update resource count")); + return; + } + hitCount = 1; + flags = 0; + } else { + PREDICTOR_LOG((" existing resource")); + hitCount = std::min(hitCount + 1, static_cast(loadCount)); + } + + // Update the rolling load count to mark this sub-resource as seen on the + // most-recent pageload so it can be eligible for prefetch (assuming all + // the other stars align). + flags |= (1 << kRollingLoadOffset); + + nsCString newValue; + MakeMetadataEntry(hitCount, lastLoad, flags, newValue); + rv = entry->SetMetaDataElement(key.BeginReading(), newValue.BeginReading()); + PREDICTOR_LOG((" SetMetaDataElement -> 0x%08X", rv)); + if (NS_FAILED(rv) && isNewResource) { + // Roll back the increment to the resource count we made above. + PREDICTOR_LOG((" rolling back resource count update")); + --resourceCount; + if (resourceCount == 0) { + entry->SetMetaDataElement(RESOURCE_META_DATA, nullptr); + } else { + nsAutoCString count; + count.AppendInt(resourceCount); + entry->SetMetaDataElement(RESOURCE_META_DATA, count.BeginReading()); + } + } +} + +// This is called when a top-level loaded ended up redirecting to a different +// URI so we can keep track of that fact. +void +Predictor::LearnForRedirect(nsICacheEntry *entry, nsIURI *targetURI) +{ + MOZ_ASSERT(NS_IsMainThread()); + + // TODO - not doing redirects for first go around + PREDICTOR_LOG(("Predictor::LearnForRedirect")); +} + +// This will add a page to our list of startup pages if it's being loaded +// before our startup window has expired. +void +Predictor::MaybeLearnForStartup(nsIURI *uri, bool fullUri) +{ + MOZ_ASSERT(NS_IsMainThread()); + + // TODO - not doing startup for first go around + PREDICTOR_LOG(("Predictor::MaybeLearnForStartup")); +} + +// Add information about a top-level load to our list of startup pages +void +Predictor::LearnForStartup(nsICacheEntry *entry, nsIURI *targetURI) +{ + MOZ_ASSERT(NS_IsMainThread()); + + // These actually do the same set of work, just on different entries, so we + // can pass through to get the real work done here + PREDICTOR_LOG(("Predictor::LearnForStartup")); + LearnForSubresource(entry, targetURI); +} + +bool +Predictor::ParseMetaDataEntry(const char *key, const char *value, nsIURI **uri, + uint32_t &hitCount, uint32_t &lastHit, + uint32_t &flags) +{ + MOZ_ASSERT(NS_IsMainThread()); + + PREDICTOR_LOG(("Predictor::ParseMetaDataEntry key=%s value=%s", + key ? key : "", value)); + + const char *comma = strchr(value, ','); + if (!comma) { + PREDICTOR_LOG((" could not find first comma")); + return false; + } + + uint32_t version = static_cast(atoi(value)); + PREDICTOR_LOG((" version -> %u", version)); + + if (version != METADATA_VERSION) { + PREDICTOR_LOG((" metadata version mismatch %u != %u", version, + METADATA_VERSION)); + return false; + } + + value = comma + 1; + comma = strchr(value, ','); + if (!comma) { + PREDICTOR_LOG((" could not find second comma")); + return false; + } + + hitCount = static_cast(atoi(value)); + PREDICTOR_LOG((" hitCount -> %u", hitCount)); + + value = comma + 1; + comma = strchr(value, ','); + if (!comma) { + PREDICTOR_LOG((" could not find third comma")); + return false; + } + + lastHit = static_cast(atoi(value)); + PREDICTOR_LOG((" lastHit -> %u", lastHit)); + + value = comma + 1; + flags = static_cast(atoi(value)); + PREDICTOR_LOG((" flags -> %u", flags)); + + if (key) { + const char *uriStart = key + (sizeof(META_DATA_PREFIX) - 1); + nsresult rv = NS_NewURI(uri, uriStart, nullptr, mIOService); + if (NS_FAILED(rv)) { + PREDICTOR_LOG((" NS_NewURI returned 0x%X", rv)); + return false; + } + PREDICTOR_LOG((" uri -> %s", uriStart)); + } + + return true; +} + +NS_IMETHODIMP +Predictor::Reset() +{ + MOZ_ASSERT(NS_IsMainThread(), + "Predictor interface methods must be called on the main thread"); + + PREDICTOR_LOG(("Predictor::Reset")); + + if (IsNeckoChild()) { + MOZ_DIAGNOSTIC_ASSERT(gNeckoChild); + + PREDICTOR_LOG((" forwarding to parent process")); + gNeckoChild->SendPredReset(); + return NS_OK; + } + + PREDICTOR_LOG((" called on parent process")); + + if (!mInitialized) { + PREDICTOR_LOG((" not initialized")); + return NS_OK; + } + + if (!mEnabled) { + PREDICTOR_LOG((" not enabled")); + return NS_OK; + } + + RefPtr reset = new Predictor::Resetter(this); + PREDICTOR_LOG((" created a resetter")); + mCacheDiskStorage->AsyncVisitStorage(reset, true); + PREDICTOR_LOG((" Cache async launched, returning now")); + + return NS_OK; +} + +NS_IMPL_ISUPPORTS(Predictor::Resetter, + nsICacheEntryOpenCallback, + nsICacheEntryMetaDataVisitor, + nsICacheStorageVisitor); + +Predictor::Resetter::Resetter(Predictor *predictor) + :mEntriesToVisit(0) + ,mPredictor(predictor) +{ } + +NS_IMETHODIMP +Predictor::Resetter::OnCacheEntryCheck(nsICacheEntry *entry, + nsIApplicationCache *appCache, + uint32_t *result) +{ + *result = nsICacheEntryOpenCallback::ENTRY_WANTED; + return NS_OK; +} + +NS_IMETHODIMP +Predictor::Resetter::OnCacheEntryAvailable(nsICacheEntry *entry, bool isNew, + nsIApplicationCache *appCache, + nsresult result) +{ + MOZ_ASSERT(NS_IsMainThread()); + + if (NS_FAILED(result)) { + // This can happen when we've tried to open an entry that doesn't exist for + // some non-reset operation, and then get reset shortly thereafter (as + // happens in some of our tests). + --mEntriesToVisit; + if (!mEntriesToVisit) { + Complete(); + } + return NS_OK; + } + + entry->VisitMetaData(this); + nsTArray keysToDelete; + keysToDelete.SwapElements(mKeysToDelete); + + for (size_t i = 0; i < keysToDelete.Length(); ++i) { + const char *key = keysToDelete[i].BeginReading(); + entry->SetMetaDataElement(key, nullptr); + } + + --mEntriesToVisit; + if (!mEntriesToVisit) { + Complete(); + } + + return NS_OK; +} + +NS_IMETHODIMP +Predictor::Resetter::OnMetaDataElement(const char *asciiKey, + const char *asciiValue) +{ + MOZ_ASSERT(NS_IsMainThread()); + + if (!StringBeginsWith(nsDependentCString(asciiKey), + NS_LITERAL_CSTRING(META_DATA_PREFIX))) { + // Not a metadata entry we care about, carry on + return NS_OK; + } + + nsCString key; + key.AssignASCII(asciiKey); + mKeysToDelete.AppendElement(key); + + return NS_OK; +} + +NS_IMETHODIMP +Predictor::Resetter::OnCacheStorageInfo(uint32_t entryCount, uint64_t consumption, + uint64_t capacity, nsIFile *diskDirectory) +{ + MOZ_ASSERT(NS_IsMainThread()); + + return NS_OK; +} + +NS_IMETHODIMP +Predictor::Resetter::OnCacheEntryInfo(nsIURI *uri, const nsACString &idEnhance, + int64_t dataSize, int32_t fetchCount, + uint32_t lastModifiedTime, uint32_t expirationTime, + bool aPinned) +{ + MOZ_ASSERT(NS_IsMainThread()); + + // The predictor will only ever touch entries with no idEnhance ("") or an + // idEnhance of PREDICTOR_ORIGIN_EXTENSION, so we filter out any entries that + // don't match that to avoid doing extra work. + if (idEnhance.EqualsLiteral(PREDICTOR_ORIGIN_EXTENSION)) { + // This is an entry we own, so we can just doom it entirely + mPredictor->mCacheDiskStorage->AsyncDoomURI(uri, idEnhance, nullptr); + } else if (idEnhance.IsEmpty()) { + // This is an entry we don't own, so we have to be a little more careful and + // just get rid of our own metadata entries. Append it to an array of things + // to operate on and then do the operations later so we don't end up calling + // Complete() multiple times/too soon. + ++mEntriesToVisit; + mURIsToVisit.AppendElement(uri); + } + + return NS_OK; +} + +NS_IMETHODIMP +Predictor::Resetter::OnCacheEntryVisitCompleted() +{ + MOZ_ASSERT(NS_IsMainThread()); + + nsTArray> urisToVisit; + urisToVisit.SwapElements(mURIsToVisit); + + MOZ_ASSERT(mEntriesToVisit == urisToVisit.Length()); + if (!mEntriesToVisit) { + Complete(); + return NS_OK; + } + + uint32_t entriesToVisit = urisToVisit.Length(); + for (uint32_t i = 0; i < entriesToVisit; ++i) { + nsCString u; + urisToVisit[i]->GetAsciiSpec(u); + mPredictor->mCacheDiskStorage->AsyncOpenURI( + urisToVisit[i], EmptyCString(), + nsICacheStorage::OPEN_READONLY | nsICacheStorage::OPEN_SECRETLY | nsICacheStorage::CHECK_MULTITHREADED, + this); + } + + return NS_OK; +} + +void +Predictor::Resetter::Complete() +{ + MOZ_ASSERT(NS_IsMainThread()); + + nsCOMPtr obs = mozilla::services::GetObserverService(); + if (!obs) { + PREDICTOR_LOG(("COULD NOT GET OBSERVER SERVICE!")); + return; + } + + obs->NotifyObservers(nullptr, "predictor-reset-complete", nullptr); +} + +// Helper functions to make using the predictor easier from native code + +static nsresult +EnsureGlobalPredictor(nsINetworkPredictor **aPredictor) +{ + MOZ_ASSERT(NS_IsMainThread()); + + nsresult rv; + nsCOMPtr predictor = + do_GetService("@mozilla.org/network/predictor;1", + &rv); + NS_ENSURE_SUCCESS(rv, rv); + + predictor.forget(aPredictor); + return NS_OK; +} + +nsresult +PredictorPredict(nsIURI *targetURI, nsIURI *sourceURI, + PredictorPredictReason reason, nsILoadContext *loadContext, + nsINetworkPredictorVerifier *verifier) +{ + MOZ_ASSERT(NS_IsMainThread()); + + if (!IsNullOrHttp(targetURI) || !IsNullOrHttp(sourceURI)) { + return NS_OK; + } + + nsCOMPtr predictor; + nsresult rv = EnsureGlobalPredictor(getter_AddRefs(predictor)); + NS_ENSURE_SUCCESS(rv, rv); + + return predictor->Predict(targetURI, sourceURI, reason, + loadContext, verifier); +} + +nsresult +PredictorLearn(nsIURI *targetURI, nsIURI *sourceURI, + PredictorLearnReason reason, + nsILoadContext *loadContext) +{ + MOZ_ASSERT(NS_IsMainThread()); + + if (!IsNullOrHttp(targetURI) || !IsNullOrHttp(sourceURI)) { + return NS_OK; + } + + nsCOMPtr predictor; + nsresult rv = EnsureGlobalPredictor(getter_AddRefs(predictor)); + NS_ENSURE_SUCCESS(rv, rv); + + return predictor->Learn(targetURI, sourceURI, reason, loadContext); +} + +nsresult +PredictorLearn(nsIURI *targetURI, nsIURI *sourceURI, + PredictorLearnReason reason, + nsILoadGroup *loadGroup) +{ + MOZ_ASSERT(NS_IsMainThread()); + + if (!IsNullOrHttp(targetURI) || !IsNullOrHttp(sourceURI)) { + return NS_OK; + } + + nsCOMPtr predictor; + nsresult rv = EnsureGlobalPredictor(getter_AddRefs(predictor)); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr loadContext; + + if (loadGroup) { + nsCOMPtr callbacks; + loadGroup->GetNotificationCallbacks(getter_AddRefs(callbacks)); + if (callbacks) { + loadContext = do_GetInterface(callbacks); + } + } + + return predictor->Learn(targetURI, sourceURI, reason, loadContext); +} + +nsresult +PredictorLearn(nsIURI *targetURI, nsIURI *sourceURI, + PredictorLearnReason reason, + nsIDocument *document) +{ + MOZ_ASSERT(NS_IsMainThread()); + + if (!IsNullOrHttp(targetURI) || !IsNullOrHttp(sourceURI)) { + return NS_OK; + } + + nsCOMPtr predictor; + nsresult rv = EnsureGlobalPredictor(getter_AddRefs(predictor)); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr loadContext; + + if (document) { + loadContext = document->GetLoadContext(); + } + + return predictor->Learn(targetURI, sourceURI, reason, loadContext); +} + +nsresult +PredictorLearnRedirect(nsIURI *targetURI, nsIChannel *channel, + nsILoadContext *loadContext) +{ + MOZ_ASSERT(NS_IsMainThread()); + + nsCOMPtr sourceURI; + nsresult rv = channel->GetOriginalURI(getter_AddRefs(sourceURI)); + NS_ENSURE_SUCCESS(rv, rv); + + bool sameUri; + rv = targetURI->Equals(sourceURI, &sameUri); + NS_ENSURE_SUCCESS(rv, rv); + + if (sameUri) { + return NS_OK; + } + + if (!IsNullOrHttp(targetURI) || !IsNullOrHttp(sourceURI)) { + return NS_OK; + } + + nsCOMPtr predictor; + rv = EnsureGlobalPredictor(getter_AddRefs(predictor)); + NS_ENSURE_SUCCESS(rv, rv); + + return predictor->Learn(targetURI, sourceURI, + nsINetworkPredictor::LEARN_LOAD_REDIRECT, + loadContext); +} + +// nsINetworkPredictorVerifier + +/** + * Call through to the child's verifier (only during tests) + */ +NS_IMETHODIMP +Predictor::OnPredictPrefetch(nsIURI *aURI, uint32_t httpStatus) +{ + if (IsNeckoChild()) { + if (mChildVerifier) { + // Ideally, we'd assert here. But since we're slowly moving towards a + // world where we have multiple child processes, and only one child process + // will be likely to have a verifier, we have to play it safer. + return mChildVerifier->OnPredictPrefetch(aURI, httpStatus); + } + return NS_OK; + } + + ipc::URIParams serURI; + SerializeURI(aURI, serURI); + + for (auto* cp : ContentParent::AllProcesses(ContentParent::eLive)) { + PNeckoParent* neckoParent = SingleManagedOrNull(cp->ManagedPNeckoParent()); + if (!neckoParent) { + continue; + } + if (!neckoParent->SendPredOnPredictPrefetch(serURI, httpStatus)) { + return NS_ERROR_NOT_AVAILABLE; + } + } + + return NS_OK; +} + +NS_IMETHODIMP +Predictor::OnPredictPreconnect(nsIURI *aURI) { + if (IsNeckoChild()) { + if (mChildVerifier) { + // Ideally, we'd assert here. But since we're slowly moving towards a + // world where we have multiple child processes, and only one child process + // will be likely to have a verifier, we have to play it safer. + return mChildVerifier->OnPredictPreconnect(aURI); + } + return NS_OK; + } + + ipc::URIParams serURI; + SerializeURI(aURI, serURI); + + for (auto* cp : ContentParent::AllProcesses(ContentParent::eLive)) { + PNeckoParent* neckoParent = SingleManagedOrNull(cp->ManagedPNeckoParent()); + if (!neckoParent) { + continue; + } + if (!neckoParent->SendPredOnPredictPreconnect(serURI)) { + return NS_ERROR_NOT_AVAILABLE; + } + } + + return NS_OK; +} + +NS_IMETHODIMP +Predictor::OnPredictDNS(nsIURI *aURI) { + if (IsNeckoChild()) { + if (mChildVerifier) { + // Ideally, we'd assert here. But since we're slowly moving towards a + // world where we have multiple child processes, and only one child process + // will be likely to have a verifier, we have to play it safer. + return mChildVerifier->OnPredictDNS(aURI); + } + return NS_OK; + } + + ipc::URIParams serURI; + SerializeURI(aURI, serURI); + + for (auto* cp : ContentParent::AllProcesses(ContentParent::eLive)) { + PNeckoParent* neckoParent = SingleManagedOrNull(cp->ManagedPNeckoParent()); + if (!neckoParent) { + continue; + } + if (!neckoParent->SendPredOnPredictDNS(serURI)) { + return NS_ERROR_NOT_AVAILABLE; + } + } + + return NS_OK; +} + +// Predictor::PrefetchListener +// nsISupports +NS_IMPL_ISUPPORTS(Predictor::PrefetchListener, + nsIStreamListener, + nsIRequestObserver) + +// nsIRequestObserver +NS_IMETHODIMP +Predictor::PrefetchListener::OnStartRequest(nsIRequest *aRequest, + nsISupports *aContext) +{ + mStartTime = TimeStamp::Now(); + return NS_OK; +} + +NS_IMETHODIMP +Predictor::PrefetchListener::OnStopRequest(nsIRequest *aRequest, + nsISupports *aContext, + nsresult aStatusCode) +{ + PREDICTOR_LOG(("OnStopRequest this=%p aStatusCode=0x%X", this, aStatusCode)); + NS_ENSURE_ARG(aRequest); + if (NS_FAILED(aStatusCode)) { + return aStatusCode; + } + Telemetry::AccumulateTimeDelta(Telemetry::PREDICTOR_PREFETCH_TIME, mStartTime); + + nsCOMPtr httpChannel = do_QueryInterface(aRequest); + if (!httpChannel) { + PREDICTOR_LOG((" Could not get HTTP Channel!")); + return NS_ERROR_UNEXPECTED; + } + nsCOMPtr cachingChannel = do_QueryInterface(httpChannel); + if (!cachingChannel) { + PREDICTOR_LOG((" Could not get caching channel!")); + return NS_ERROR_UNEXPECTED; + } + + nsresult rv = NS_OK; + uint32_t httpStatus; + rv = httpChannel->GetResponseStatus(&httpStatus); + if (NS_SUCCEEDED(rv) && httpStatus == 200) { + rv = cachingChannel->ForceCacheEntryValidFor(mPredictor->mPrefetchForceValidFor); + PREDICTOR_LOG((" forcing entry valid for %d seconds rv=%X", + mPredictor->mPrefetchForceValidFor, rv)); + } else { + rv = cachingChannel->ForceCacheEntryValidFor(0); + PREDICTOR_LOG((" removing any forced validity rv=%X", rv)); + } + + nsAutoCString reqName; + rv = aRequest->GetName(reqName); + if (NS_FAILED(rv)) { + reqName.AssignLiteral(""); + } + + PREDICTOR_LOG((" request %s status %u", reqName.get(), httpStatus)); + + if (mVerifier) { + mVerifier->OnPredictPrefetch(mURI, httpStatus); + } + + return rv; +} + +// nsIStreamListener +NS_IMETHODIMP +Predictor::PrefetchListener::OnDataAvailable(nsIRequest *aRequest, + nsISupports *aContext, + nsIInputStream *aInputStream, + uint64_t aOffset, + const uint32_t aCount) +{ + uint32_t result; + return aInputStream->ReadSegments(NS_DiscardSegment, nullptr, aCount, &result); +} + +// Miscellaneous Predictor + +void +Predictor::UpdateCacheability(nsIURI *sourceURI, nsIURI *targetURI, + uint32_t httpStatus, + nsHttpRequestHead &requestHead, + nsHttpResponseHead *responseHead, + nsILoadContextInfo *lci) +{ + MOZ_ASSERT(NS_IsMainThread()); + + if (lci && lci->IsPrivate()) { + PREDICTOR_LOG(("Predictor::UpdateCacheability in PB mode - ignoring")); + return; + } + + if (!sourceURI || !targetURI) { + PREDICTOR_LOG(("Predictor::UpdateCacheability missing source or target uri")); + return; + } + + if (!IsNullOrHttp(sourceURI) || !IsNullOrHttp(targetURI)) { + PREDICTOR_LOG(("Predictor::UpdateCacheability non-http(s) uri")); + return; + } + + RefPtr self = sSelf; + if (self) { + nsAutoCString method; + requestHead.Method(method); + self->UpdateCacheabilityInternal(sourceURI, targetURI, httpStatus, + method); + } +} + +void +Predictor::UpdateCacheabilityInternal(nsIURI *sourceURI, nsIURI *targetURI, + uint32_t httpStatus, + const nsCString &method) +{ + PREDICTOR_LOG(("Predictor::UpdateCacheability httpStatus=%u", httpStatus)); + + if (!mInitialized) { + PREDICTOR_LOG((" not initialized")); + return; + } + + if (!mEnabled) { + PREDICTOR_LOG((" not enabled")); + return; + } + + if (!mEnablePrefetch) { + PREDICTOR_LOG((" prefetch not enabled")); + return; + } + + uint32_t openFlags = nsICacheStorage::OPEN_READONLY | + nsICacheStorage::OPEN_SECRETLY | + nsICacheStorage::CHECK_MULTITHREADED; + RefPtr action = + new Predictor::CacheabilityAction(targetURI, httpStatus, method, this); + nsAutoCString uri; + targetURI->GetAsciiSpec(uri); + PREDICTOR_LOG((" uri=%s action=%p", uri.get(), action.get())); + mCacheDiskStorage->AsyncOpenURI(sourceURI, EmptyCString(), openFlags, action); +} + +NS_IMPL_ISUPPORTS(Predictor::CacheabilityAction, + nsICacheEntryOpenCallback, + nsICacheEntryMetaDataVisitor); + +NS_IMETHODIMP +Predictor::CacheabilityAction::OnCacheEntryCheck(nsICacheEntry *entry, + nsIApplicationCache *appCache, + uint32_t *result) +{ + *result = nsICacheEntryOpenCallback::ENTRY_WANTED; + return NS_OK; +} + +NS_IMETHODIMP +Predictor::CacheabilityAction::OnCacheEntryAvailable(nsICacheEntry *entry, + bool isNew, + nsIApplicationCache *appCache, + nsresult result) +{ + MOZ_ASSERT(NS_IsMainThread()); + // This is being opened read-only, so isNew should always be false + MOZ_ASSERT(!isNew); + + PREDICTOR_LOG(("CacheabilityAction::OnCacheEntryAvailable this=%p", this)); + if (NS_FAILED(result)) { + // Nothing to do + PREDICTOR_LOG((" nothing to do result=%X isNew=%d", result, isNew)); + return NS_OK; + } + + nsresult rv = entry->VisitMetaData(this); + if (NS_FAILED(rv)) { + PREDICTOR_LOG((" VisitMetaData returned %x", rv)); + return NS_OK; + } + + nsTArray keysToCheck, valuesToCheck; + keysToCheck.SwapElements(mKeysToCheck); + valuesToCheck.SwapElements(mValuesToCheck); + + MOZ_ASSERT(keysToCheck.Length() == valuesToCheck.Length()); + for (size_t i = 0; i < keysToCheck.Length(); ++i) { + const char *key = keysToCheck[i].BeginReading(); + const char *value = valuesToCheck[i].BeginReading(); + nsCOMPtr uri; + uint32_t hitCount, lastHit, flags; + + if (!mPredictor->ParseMetaDataEntry(key, value, getter_AddRefs(uri), + hitCount, lastHit, flags)) { + PREDICTOR_LOG((" failed to parse key=%s value=%s", key, value)); + continue; + } + + bool eq = false; + if (NS_SUCCEEDED(uri->Equals(mTargetURI, &eq)) && eq) { + if (mHttpStatus == 200 && mMethod.EqualsLiteral("GET")) { + PREDICTOR_LOG((" marking %s cacheable", key)); + flags |= FLAG_PREFETCHABLE; + } else { + PREDICTOR_LOG((" marking %s uncacheable", key)); + flags &= ~FLAG_PREFETCHABLE; + } + nsCString newValue; + MakeMetadataEntry(hitCount, lastHit, flags, newValue); + entry->SetMetaDataElement(key, newValue.BeginReading()); + break; + } + } + + return NS_OK; +} + +NS_IMETHODIMP +Predictor::CacheabilityAction::OnMetaDataElement(const char *asciiKey, + const char *asciiValue) +{ + MOZ_ASSERT(NS_IsMainThread()); + + if (!IsURIMetadataElement(asciiKey)) { + return NS_OK; + } + + nsCString key, value; + key.AssignASCII(asciiKey); + value.AssignASCII(asciiValue); + mKeysToCheck.AppendElement(key); + mValuesToCheck.AppendElement(value); + + return NS_OK; +} + +} // namespace net +} // namespace mozilla -- cgit v1.2.3