diff options
Diffstat (limited to 'toolkit/components/url-classifier/nsUrlClassifierDBService.cpp')
| -rw-r--r-- | toolkit/components/url-classifier/nsUrlClassifierDBService.cpp | 1866 |
1 files changed, 1866 insertions, 0 deletions
diff --git a/toolkit/components/url-classifier/nsUrlClassifierDBService.cpp b/toolkit/components/url-classifier/nsUrlClassifierDBService.cpp new file mode 100644 index 000000000..2ad8b6b51 --- /dev/null +++ b/toolkit/components/url-classifier/nsUrlClassifierDBService.cpp @@ -0,0 +1,1866 @@ +//* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +#include "nsAutoPtr.h" +#include "nsCOMPtr.h" +#include "nsAppDirectoryServiceDefs.h" +#include "nsCRT.h" +#include "nsICryptoHash.h" +#include "nsICryptoHMAC.h" +#include "nsIDirectoryService.h" +#include "nsIKeyModule.h" +#include "nsIObserverService.h" +#include "nsIPermissionManager.h" +#include "nsIPrefBranch.h" +#include "nsIPrefService.h" +#include "nsIProperties.h" +#include "nsToolkitCompsCID.h" +#include "nsIUrlClassifierUtils.h" +#include "nsIXULRuntime.h" +#include "nsUrlClassifierDBService.h" +#include "nsUrlClassifierUtils.h" +#include "nsUrlClassifierProxies.h" +#include "nsURILoader.h" +#include "nsString.h" +#include "nsReadableUtils.h" +#include "nsTArray.h" +#include "nsNetCID.h" +#include "nsThreadUtils.h" +#include "nsXPCOMStrings.h" +#include "nsProxyRelease.h" +#include "nsString.h" +#include "mozilla/Atomics.h" +#include "mozilla/DebugOnly.h" +#include "mozilla/ErrorNames.h" +#include "mozilla/Mutex.h" +#include "mozilla/Preferences.h" +#include "mozilla/TimeStamp.h" +#include "mozilla/Telemetry.h" +#include "mozilla/Logging.h" +#include "prprf.h" +#include "prnetdb.h" +#include "Entries.h" +#include "HashStore.h" +#include "Classifier.h" +#include "ProtocolParser.h" +#include "mozilla/Attributes.h" +#include "nsIPrincipal.h" +#include "Classifier.h" +#include "ProtocolParser.h" +#include "nsContentUtils.h" + +namespace mozilla { +namespace safebrowsing { + +nsresult +TablesToResponse(const nsACString& tables) +{ + if (tables.IsEmpty()) { + return NS_OK; + } + + // We don't check mCheckMalware and friends because BuildTables never + // includes a table that is not enabled. + if (FindInReadable(NS_LITERAL_CSTRING("-malware-"), tables)) { + return NS_ERROR_MALWARE_URI; + } + if (FindInReadable(NS_LITERAL_CSTRING("-phish-"), tables)) { + return NS_ERROR_PHISHING_URI; + } + if (FindInReadable(NS_LITERAL_CSTRING("-unwanted-"), tables)) { + return NS_ERROR_UNWANTED_URI; + } + if (FindInReadable(NS_LITERAL_CSTRING("-track-"), tables)) { + return NS_ERROR_TRACKING_URI; + } + if (FindInReadable(NS_LITERAL_CSTRING("-block-"), tables)) { + return NS_ERROR_BLOCKED_URI; + } + return NS_OK; +} + +} // namespace safebrowsing +} // namespace mozilla + +using namespace mozilla; +using namespace mozilla::safebrowsing; + +// MOZ_LOG=UrlClassifierDbService:5 +LazyLogModule gUrlClassifierDbServiceLog("UrlClassifierDbService"); +#define LOG(args) MOZ_LOG(gUrlClassifierDbServiceLog, mozilla::LogLevel::Debug, args) +#define LOG_ENABLED() MOZ_LOG_TEST(gUrlClassifierDbServiceLog, mozilla::LogLevel::Debug) + +// Prefs for implementing nsIURIClassifier to block page loads +#define CHECK_MALWARE_PREF "browser.safebrowsing.malware.enabled" +#define CHECK_MALWARE_DEFAULT false + +#define CHECK_PHISHING_PREF "browser.safebrowsing.phishing.enabled" +#define CHECK_PHISHING_DEFAULT false + +#define CHECK_TRACKING_PREF "privacy.trackingprotection.enabled" +#define CHECK_TRACKING_DEFAULT false + +#define CHECK_TRACKING_PB_PREF "privacy.trackingprotection.pbmode.enabled" +#define CHECK_TRACKING_PB_DEFAULT false + +#define CHECK_BLOCKED_PREF "browser.safebrowsing.blockedURIs.enabled" +#define CHECK_BLOCKED_DEFAULT false + +#define GETHASH_NOISE_PREF "urlclassifier.gethashnoise" +#define GETHASH_NOISE_DEFAULT 4 + +// Comma-separated lists +#define MALWARE_TABLE_PREF "urlclassifier.malwareTable" +#define PHISH_TABLE_PREF "urlclassifier.phishTable" +#define TRACKING_TABLE_PREF "urlclassifier.trackingTable" +#define TRACKING_WHITELIST_TABLE_PREF "urlclassifier.trackingWhitelistTable" +#define BLOCKED_TABLE_PREF "urlclassifier.blockedTable" +#define DOWNLOAD_BLOCK_TABLE_PREF "urlclassifier.downloadBlockTable" +#define DOWNLOAD_ALLOW_TABLE_PREF "urlclassifier.downloadAllowTable" +#define DISALLOW_COMPLETION_TABLE_PREF "urlclassifier.disallow_completions" + +#define CONFIRM_AGE_PREF "urlclassifier.max-complete-age" +#define CONFIRM_AGE_DEFAULT_SEC (45 * 60) + +class nsUrlClassifierDBServiceWorker; + +// Singleton instance. +static nsUrlClassifierDBService* sUrlClassifierDBService; + +nsIThread* nsUrlClassifierDBService::gDbBackgroundThread = nullptr; + +// Once we've committed to shutting down, don't do work in the background +// thread. +static bool gShuttingDownThread = false; + +static mozilla::Atomic<int32_t> gFreshnessGuarantee(CONFIRM_AGE_DEFAULT_SEC); + +NS_IMPL_ISUPPORTS(nsUrlClassifierDBServiceWorker, + nsIUrlClassifierDBService) + +nsUrlClassifierDBServiceWorker::nsUrlClassifierDBServiceWorker() + : mInStream(false) + , mGethashNoise(0) + , mPendingLookupLock("nsUrlClassifierDBServerWorker.mPendingLookupLock") +{ +} + +nsUrlClassifierDBServiceWorker::~nsUrlClassifierDBServiceWorker() +{ + NS_ASSERTION(!mClassifier, + "Db connection not closed, leaking memory! Call CloseDb " + "to close the connection."); +} + +nsresult +nsUrlClassifierDBServiceWorker::Init(uint32_t aGethashNoise, + nsCOMPtr<nsIFile> aCacheDir) +{ + mGethashNoise = aGethashNoise; + mCacheDir = aCacheDir; + + ResetUpdate(); + + return NS_OK; +} + +nsresult +nsUrlClassifierDBServiceWorker::QueueLookup(const nsACString& spec, + const nsACString& tables, + nsIUrlClassifierLookupCallback* callback) +{ + MutexAutoLock lock(mPendingLookupLock); + + PendingLookup* lookup = mPendingLookups.AppendElement(); + if (!lookup) return NS_ERROR_OUT_OF_MEMORY; + + lookup->mStartTime = TimeStamp::Now(); + lookup->mKey = spec; + lookup->mCallback = callback; + lookup->mTables = tables; + + return NS_OK; +} + +nsresult +nsUrlClassifierDBServiceWorker::DoLocalLookup(const nsACString& spec, + const nsACString& tables, + LookupResultArray* results) +{ + if (gShuttingDownThread) { + return NS_ERROR_ABORT; + } + + MOZ_ASSERT(!NS_IsMainThread(), "DoLocalLookup must be on background thread"); + if (!results) { + return NS_ERROR_FAILURE; + } + // Bail if we haven't been initialized on the background thread. + if (!mClassifier) { + return NS_ERROR_NOT_AVAILABLE; + } + + // We ignore failures from Check because we'd rather return the + // results that were found than fail. + mClassifier->Check(spec, tables, gFreshnessGuarantee, *results); + + LOG(("Found %d results.", results->Length())); + return NS_OK; +} + +static nsCString +ProcessLookupResults(LookupResultArray* results) +{ + // Build a stringified list of result tables. + nsTArray<nsCString> tables; + for (uint32_t i = 0; i < results->Length(); i++) { + LookupResult& result = results->ElementAt(i); + MOZ_ASSERT(!result.mNoise, "Lookup results should not have noise added"); + LOG(("Found result from table %s", result.mTableName.get())); + if (tables.IndexOf(result.mTableName) == nsTArray<nsCString>::NoIndex) { + tables.AppendElement(result.mTableName); + } + } + nsAutoCString tableStr; + for (uint32_t i = 0; i < tables.Length(); i++) { + if (i != 0) + tableStr.Append(','); + tableStr.Append(tables[i]); + } + return tableStr; +} + +/** + * Lookup up a key in the database is a two step process: + * + * a) First we look for any Entries in the database that might apply to this + * url. For each URL there are one or two possible domain names to check: + * the two-part domain name (example.com) and the three-part name + * (www.example.com). We check the database for both of these. + * b) If we find any entries, we check the list of fragments for that entry + * against the possible subfragments of the URL as described in the + * "Simplified Regular Expression Lookup" section of the protocol doc. + */ +nsresult +nsUrlClassifierDBServiceWorker::DoLookup(const nsACString& spec, + const nsACString& tables, + nsIUrlClassifierLookupCallback* c) +{ + if (gShuttingDownThread) { + c->LookupComplete(nullptr); + return NS_ERROR_NOT_INITIALIZED; + } + + PRIntervalTime clockStart = 0; + if (LOG_ENABLED()) { + clockStart = PR_IntervalNow(); + } + + nsAutoPtr<LookupResultArray> results(new LookupResultArray()); + if (!results) { + c->LookupComplete(nullptr); + return NS_ERROR_OUT_OF_MEMORY; + } + + nsresult rv = DoLocalLookup(spec, tables, results); + if (NS_FAILED(rv)) { + c->LookupComplete(nullptr); + return rv; + } + + LOG(("Found %d results.", results->Length())); + + + if (LOG_ENABLED()) { + PRIntervalTime clockEnd = PR_IntervalNow(); + LOG(("query took %dms\n", + PR_IntervalToMilliseconds(clockEnd - clockStart))); + } + + nsAutoPtr<LookupResultArray> completes(new LookupResultArray()); + + for (uint32_t i = 0; i < results->Length(); i++) { + if (!mMissCache.Contains(results->ElementAt(i).hash.prefix)) { + completes->AppendElement(results->ElementAt(i)); + } + } + + for (uint32_t i = 0; i < completes->Length(); i++) { + if (!completes->ElementAt(i).Confirmed()) { + // We're going to be doing a gethash request, add some extra entries. + // Note that we cannot pass the first two by reference, because we + // add to completes, whicah can cause completes to reallocate and move. + AddNoise(completes->ElementAt(i).hash.prefix, + completes->ElementAt(i).mTableName, + mGethashNoise, *completes); + break; + } + } + + // At this point ownership of 'results' is handed to the callback. + c->LookupComplete(completes.forget()); + + return NS_OK; +} + +nsresult +nsUrlClassifierDBServiceWorker::HandlePendingLookups() +{ + if (gShuttingDownThread) { + return NS_ERROR_ABORT; + } + + MutexAutoLock lock(mPendingLookupLock); + while (mPendingLookups.Length() > 0) { + PendingLookup lookup = mPendingLookups[0]; + mPendingLookups.RemoveElementAt(0); + { + MutexAutoUnlock unlock(mPendingLookupLock); + DoLookup(lookup.mKey, lookup.mTables, lookup.mCallback); + } + double lookupTime = (TimeStamp::Now() - lookup.mStartTime).ToMilliseconds(); + Telemetry::Accumulate(Telemetry::URLCLASSIFIER_LOOKUP_TIME, + static_cast<uint32_t>(lookupTime)); + } + + return NS_OK; +} + +nsresult +nsUrlClassifierDBServiceWorker::AddNoise(const Prefix aPrefix, + const nsCString tableName, + uint32_t aCount, + LookupResultArray& results) +{ + if (gShuttingDownThread) { + return NS_ERROR_ABORT; + } + + if (aCount < 1) { + return NS_OK; + } + + PrefixArray noiseEntries; + nsresult rv = mClassifier->ReadNoiseEntries(aPrefix, tableName, + aCount, &noiseEntries); + NS_ENSURE_SUCCESS(rv, rv); + + for (uint32_t i = 0; i < noiseEntries.Length(); i++) { + LookupResult *result = results.AppendElement(); + if (!result) + return NS_ERROR_OUT_OF_MEMORY; + + result->hash.prefix = noiseEntries[i]; + result->mNoise = true; + + result->mTableName.Assign(tableName); + } + + return NS_OK; +} + +// Lookup a key in the db. +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::Lookup(nsIPrincipal* aPrincipal, + const nsACString& aTables, + nsIUrlClassifierCallback* c) +{ + if (gShuttingDownThread) { + return NS_ERROR_ABORT; + } + + return HandlePendingLookups(); +} + +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::GetTables(nsIUrlClassifierCallback* c) +{ + if (gShuttingDownThread) { + return NS_ERROR_NOT_INITIALIZED; + } + + nsresult rv = OpenDb(); + if (NS_FAILED(rv)) { + NS_ERROR("Unable to open SafeBrowsing database"); + return NS_ERROR_FAILURE; + } + + NS_ENSURE_SUCCESS(rv, rv); + + nsAutoCString response; + mClassifier->TableRequest(response); + LOG(("GetTables: %s", response.get())); + c->HandleEvent(response); + + return rv; +} + +void +nsUrlClassifierDBServiceWorker::ResetStream() +{ + LOG(("ResetStream")); + mInStream = false; + mProtocolParser = nullptr; +} + +void +nsUrlClassifierDBServiceWorker::ResetUpdate() +{ + LOG(("ResetUpdate")); + mUpdateWaitSec = 0; + mUpdateStatus = NS_OK; + mUpdateObserver = nullptr; +} + +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::SetHashCompleter(const nsACString &tableName, + nsIUrlClassifierHashCompleter *completer) +{ + return NS_ERROR_NOT_IMPLEMENTED; +} + +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::BeginUpdate(nsIUrlClassifierUpdateObserver *observer, + const nsACString &tables) +{ + LOG(("nsUrlClassifierDBServiceWorker::BeginUpdate [%s]", PromiseFlatCString(tables).get())); + + if (gShuttingDownThread) { + return NS_ERROR_NOT_INITIALIZED; + } + + NS_ENSURE_STATE(!mUpdateObserver); + + nsresult rv = OpenDb(); + if (NS_FAILED(rv)) { + NS_ERROR("Unable to open SafeBrowsing database"); + return NS_ERROR_FAILURE; + } + + mUpdateStatus = NS_OK; + mUpdateObserver = observer; + Classifier::SplitTables(tables, mUpdateTables); + + return NS_OK; +} + +// Called from the stream updater. +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::BeginStream(const nsACString &table) +{ + LOG(("nsUrlClassifierDBServiceWorker::BeginStream")); + MOZ_ASSERT(!NS_IsMainThread(), "Streaming must be on the background thread"); + + if (gShuttingDownThread) { + return NS_ERROR_NOT_INITIALIZED; + } + + NS_ENSURE_STATE(mUpdateObserver); + NS_ENSURE_STATE(!mInStream); + + mInStream = true; + + NS_ASSERTION(!mProtocolParser, "Should not have a protocol parser."); + + // Check if we should use protobuf to parse the update. + bool useProtobuf = false; + for (size_t i = 0; i < mUpdateTables.Length(); i++) { + bool isCurProtobuf = + StringEndsWith(mUpdateTables[i], NS_LITERAL_CSTRING("-proto")); + + if (0 == i) { + // Use the first table name to decice if all the subsequent tables + // should be '-proto'. + useProtobuf = isCurProtobuf; + continue; + } + + if (useProtobuf != isCurProtobuf) { + NS_WARNING("Cannot mix 'proto' tables with other types " + "within the same provider."); + break; + } + } + + mProtocolParser = (useProtobuf ? static_cast<ProtocolParser*>(new ProtocolParserProtobuf()) + : static_cast<ProtocolParser*>(new ProtocolParserV2())); + if (!mProtocolParser) + return NS_ERROR_OUT_OF_MEMORY; + + mProtocolParser->Init(mCryptoHash); + + if (!table.IsEmpty()) { + mProtocolParser->SetCurrentTable(table); + } + + mProtocolParser->SetRequestedTables(mUpdateTables); + + return NS_OK; +} + +/** + * Updating the database: + * + * The Update() method takes a series of chunks separated with control data, + * as described in + * http://code.google.com/p/google-safe-browsing/wiki/Protocolv2Spec + * + * It will iterate through the control data until it reaches a chunk. By + * the time it reaches a chunk, it should have received + * a) the table to which this chunk applies + * b) the type of chunk (add, delete, expire add, expire delete). + * c) the chunk ID + * d) the length of the chunk. + * + * For add and subtract chunks, it needs to read the chunk data (expires + * don't have any data). Chunk data is a list of URI fragments whose + * encoding depends on the type of table (which is indicated by the end + * of the table name): + * a) tables ending with -exp are a zlib-compressed list of URI fragments + * separated by newlines. + * b) tables ending with -sha128 have the form + * [domain][N][frag0]...[fragN] + * 16 1 16 16 + * If N is 0, the domain is reused as a fragment. + * c) any other tables are assumed to be a plaintext list of URI fragments + * separated by newlines. + * + * Update() can be fed partial data; It will accumulate data until there is + * enough to act on. Finish() should be called when there will be no more + * data. + */ +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::UpdateStream(const nsACString& chunk) +{ + if (gShuttingDownThread) { + return NS_ERROR_NOT_INITIALIZED; + } + + NS_ENSURE_STATE(mInStream); + + HandlePendingLookups(); + + // Feed the chunk to the parser. + return mProtocolParser->AppendStream(chunk); +} + +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::FinishStream() +{ + if (gShuttingDownThread) { + LOG(("shutting down")); + return NS_ERROR_NOT_INITIALIZED; + } + + NS_ENSURE_STATE(mInStream); + NS_ENSURE_STATE(mUpdateObserver); + + mInStream = false; + + mProtocolParser->End(); + + if (NS_SUCCEEDED(mProtocolParser->Status())) { + if (mProtocolParser->UpdateWaitSec()) { + mUpdateWaitSec = mProtocolParser->UpdateWaitSec(); + } + // XXX: Only allow forwards from the initial update? + const nsTArray<ProtocolParser::ForwardedUpdate> &forwards = + mProtocolParser->Forwards(); + for (uint32_t i = 0; i < forwards.Length(); i++) { + const ProtocolParser::ForwardedUpdate &forward = forwards[i]; + mUpdateObserver->UpdateUrlRequested(forward.url, forward.table); + } + // Hold on to any TableUpdate objects that were created by the + // parser. + mTableUpdates.AppendElements(mProtocolParser->GetTableUpdates()); + mProtocolParser->ForgetTableUpdates(); + +#ifdef MOZ_SAFEBROWSING_DUMP_FAILED_UPDATES + // The assignment involves no string copy since the source string is sharable. + mRawTableUpdates = mProtocolParser->GetRawTableUpdates(); +#endif + } else { + LOG(("nsUrlClassifierDBService::FinishStream Failed to parse the stream " + "using mProtocolParser.")); + mUpdateStatus = mProtocolParser->Status(); + } + mUpdateObserver->StreamFinished(mProtocolParser->Status(), 0); + + if (NS_SUCCEEDED(mUpdateStatus)) { + if (mProtocolParser->ResetRequested()) { + mClassifier->ResetTables(Classifier::Clear_All, mUpdateTables); + } + } + + mProtocolParser = nullptr; + + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::FinishUpdate() +{ + if (gShuttingDownThread) { + return NS_ERROR_NOT_INITIALIZED; + } + + NS_ENSURE_STATE(mUpdateObserver); + + if (NS_SUCCEEDED(mUpdateStatus)) { + mUpdateStatus = ApplyUpdate(); + } else { + LOG(("nsUrlClassifierDBServiceWorker::FinishUpdate() Not running " + "ApplyUpdate() since the update has already failed.")); + } + + mMissCache.Clear(); + + if (NS_SUCCEEDED(mUpdateStatus)) { + LOG(("Notifying success: %d", mUpdateWaitSec)); + mUpdateObserver->UpdateSuccess(mUpdateWaitSec); + } else if (NS_ERROR_NOT_IMPLEMENTED == mUpdateStatus) { + LOG(("Treating NS_ERROR_NOT_IMPLEMENTED a successful update " + "but still mark it spoiled.")); + mUpdateObserver->UpdateSuccess(0); + mClassifier->ResetTables(Classifier::Clear_Cache, mUpdateTables); + } else { + if (LOG_ENABLED()) { + nsAutoCString errorName; + mozilla::GetErrorName(mUpdateStatus, errorName); + LOG(("Notifying error: %s (%d)", errorName.get(), mUpdateStatus)); + } + + mUpdateObserver->UpdateError(mUpdateStatus); + /* + * mark the tables as spoiled(clear cache in LookupCache), we don't want to + * block hosts longer than normal because our update failed + */ + mClassifier->ResetTables(Classifier::Clear_Cache, mUpdateTables); + } + mUpdateObserver = nullptr; + + return NS_OK; +} + +nsresult +nsUrlClassifierDBServiceWorker::ApplyUpdate() +{ + LOG(("nsUrlClassifierDBServiceWorker::ApplyUpdate()")); + nsresult rv = mClassifier->ApplyUpdates(&mTableUpdates); + +#ifdef MOZ_SAFEBROWSING_DUMP_FAILED_UPDATES + if (NS_FAILED(rv) && NS_ERROR_OUT_OF_MEMORY != rv) { + mClassifier->DumpRawTableUpdates(mRawTableUpdates); + } + // Invalidate the raw table updates. + mRawTableUpdates = EmptyCString(); +#endif + + return rv; +} + +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::ResetDatabase() +{ + nsresult rv = OpenDb(); + + if (NS_SUCCEEDED(rv)) { + mClassifier->Reset(); + } + + rv = CloseDb(); + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::ReloadDatabase() +{ + nsTArray<nsCString> tables; + nsTArray<int64_t> lastUpdateTimes; + nsresult rv = mClassifier->ActiveTables(tables); + NS_ENSURE_SUCCESS(rv, rv); + + // We need to make sure lastupdatetime is set after reload database + // Otherwise request will be skipped if it is not confirmed. + for (uint32_t table = 0; table < tables.Length(); table++) { + lastUpdateTimes.AppendElement(mClassifier->GetLastUpdateTime(tables[table])); + } + + // This will null out mClassifier + rv = CloseDb(); + NS_ENSURE_SUCCESS(rv, rv); + + // Create new mClassifier and load prefixset and completions from disk. + rv = OpenDb(); + NS_ENSURE_SUCCESS(rv, rv); + + for (uint32_t table = 0; table < tables.Length(); table++) { + int64_t time = lastUpdateTimes[table]; + if (time) { + mClassifier->SetLastUpdateTime(tables[table], lastUpdateTimes[table]); + } + } + + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::CancelUpdate() +{ + LOG(("nsUrlClassifierDBServiceWorker::CancelUpdate")); + + if (mUpdateObserver) { + LOG(("UpdateObserver exists, cancelling")); + + mUpdateStatus = NS_BINDING_ABORTED; + + mUpdateObserver->UpdateError(mUpdateStatus); + + /* + * mark the tables as spoiled(clear cache in LookupCache), we don't want to + * block hosts longer than normal because our update failed + */ + mClassifier->ResetTables(Classifier::Clear_Cache, mUpdateTables); + + ResetStream(); + ResetUpdate(); + } else { + LOG(("No UpdateObserver, nothing to cancel")); + } + + return NS_OK; +} + +// Allows the main thread to delete the connection which may be in +// a background thread. +// XXX This could be turned into a single shutdown event so the logic +// is simpler in nsUrlClassifierDBService::Shutdown. +nsresult +nsUrlClassifierDBServiceWorker::CloseDb() +{ + if (mClassifier) { + mClassifier->Close(); + mClassifier = nullptr; + } + + mCryptoHash = nullptr; + LOG(("urlclassifier db closed\n")); + + return NS_OK; +} + +nsresult +nsUrlClassifierDBServiceWorker::CacheCompletions(CacheResultArray *results) +{ + if (gShuttingDownThread) { + return NS_ERROR_ABORT; + } + + LOG(("nsUrlClassifierDBServiceWorker::CacheCompletions [%p]", this)); + if (!mClassifier) + return NS_OK; + + // Ownership is transferred in to us + nsAutoPtr<CacheResultArray> resultsPtr(results); + + if (mLastResults == *resultsPtr) { + LOG(("Skipping completions that have just been cached already.")); + return NS_OK; + } + + nsAutoPtr<ProtocolParserV2> pParse(new ProtocolParserV2()); + nsTArray<TableUpdate*> updates; + + // Only cache results for tables that we have, don't take + // in tables we might accidentally have hit during a completion. + // This happens due to goog vs googpub lists existing. + nsTArray<nsCString> tables; + nsresult rv = mClassifier->ActiveTables(tables); + NS_ENSURE_SUCCESS(rv, rv); + + for (uint32_t i = 0; i < resultsPtr->Length(); i++) { + bool activeTable = false; + for (uint32_t table = 0; table < tables.Length(); table++) { + if (tables[table].Equals(resultsPtr->ElementAt(i).table)) { + activeTable = true; + break; + } + } + if (activeTable) { + TableUpdateV2* tuV2 = TableUpdate::Cast<TableUpdateV2>( + pParse->GetTableUpdate(resultsPtr->ElementAt(i).table)); + + NS_ENSURE_TRUE(tuV2, NS_ERROR_FAILURE); + + LOG(("CacheCompletion Addchunk %d hash %X", resultsPtr->ElementAt(i).entry.addChunk, + resultsPtr->ElementAt(i).entry.ToUint32())); + rv = tuV2->NewAddComplete(resultsPtr->ElementAt(i).entry.addChunk, + resultsPtr->ElementAt(i).entry.complete); + if (NS_FAILED(rv)) { + // We can bail without leaking here because ForgetTableUpdates + // hasn't been called yet. + return rv; + } + rv = tuV2->NewAddChunk(resultsPtr->ElementAt(i).entry.addChunk); + if (NS_FAILED(rv)) { + return rv; + } + updates.AppendElement(tuV2); + pParse->ForgetTableUpdates(); + } else { + LOG(("Completion received, but table is not active, so not caching.")); + } + } + + mClassifier->ApplyFullHashes(&updates); + mLastResults = *resultsPtr; + return NS_OK; +} + +nsresult +nsUrlClassifierDBServiceWorker::CacheMisses(PrefixArray *results) +{ + LOG(("nsUrlClassifierDBServiceWorker::CacheMisses [%p] %d", + this, results->Length())); + + // Ownership is transferred in to us + nsAutoPtr<PrefixArray> resultsPtr(results); + + for (uint32_t i = 0; i < resultsPtr->Length(); i++) { + mMissCache.AppendElement(resultsPtr->ElementAt(i)); + } + return NS_OK; +} + +nsresult +nsUrlClassifierDBServiceWorker::OpenDb() +{ + if (gShuttingDownThread) { + return NS_ERROR_ABORT; + } + + MOZ_ASSERT(!NS_IsMainThread(), "Must initialize DB on background thread"); + // Connection already open, don't do anything. + if (mClassifier) { + return NS_OK; + } + + nsresult rv; + mCryptoHash = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv); + NS_ENSURE_SUCCESS(rv, rv); + + nsAutoPtr<Classifier> classifier(new Classifier()); + if (!classifier) { + return NS_ERROR_OUT_OF_MEMORY; + } + + rv = classifier->Open(*mCacheDir); + NS_ENSURE_SUCCESS(rv, rv); + + mClassifier = classifier; + + return NS_OK; +} + +nsresult +nsUrlClassifierDBServiceWorker::SetLastUpdateTime(const nsACString &table, + uint64_t updateTime) +{ + MOZ_ASSERT(!NS_IsMainThread(), "Must be on the background thread"); + MOZ_ASSERT(mClassifier, "Classifier connection must be opened"); + + mClassifier->SetLastUpdateTime(table, updateTime); + + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierDBServiceWorker::ClearLastResults() +{ + MOZ_ASSERT(!NS_IsMainThread(), "Must be on the background thread"); + mLastResults.Clear(); + return NS_OK; +} + + +// ------------------------------------------------------------------------- +// nsUrlClassifierLookupCallback +// +// This class takes the results of a lookup found on the worker thread +// and handles any necessary partial hash expansions before calling +// the client callback. + +class nsUrlClassifierLookupCallback final : public nsIUrlClassifierLookupCallback + , public nsIUrlClassifierHashCompleterCallback +{ +public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSIURLCLASSIFIERLOOKUPCALLBACK + NS_DECL_NSIURLCLASSIFIERHASHCOMPLETERCALLBACK + + nsUrlClassifierLookupCallback(nsUrlClassifierDBService *dbservice, + nsIUrlClassifierCallback *c) + : mDBService(dbservice) + , mResults(nullptr) + , mPendingCompletions(0) + , mCallback(c) + {} + +private: + ~nsUrlClassifierLookupCallback(); + + nsresult HandleResults(); + + RefPtr<nsUrlClassifierDBService> mDBService; + nsAutoPtr<LookupResultArray> mResults; + + // Completed results to send back to the worker for caching. + nsAutoPtr<CacheResultArray> mCacheResults; + + uint32_t mPendingCompletions; + nsCOMPtr<nsIUrlClassifierCallback> mCallback; +}; + +NS_IMPL_ISUPPORTS(nsUrlClassifierLookupCallback, + nsIUrlClassifierLookupCallback, + nsIUrlClassifierHashCompleterCallback) + +nsUrlClassifierLookupCallback::~nsUrlClassifierLookupCallback() +{ + if (mCallback) { + NS_ReleaseOnMainThread(mCallback.forget()); + } +} + +NS_IMETHODIMP +nsUrlClassifierLookupCallback::LookupComplete(nsTArray<LookupResult>* results) +{ + NS_ASSERTION(mResults == nullptr, + "Should only get one set of results per nsUrlClassifierLookupCallback!"); + + if (!results) { + HandleResults(); + return NS_OK; + } + + mResults = results; + + // Check the results entries that need to be completed. + for (uint32_t i = 0; i < results->Length(); i++) { + LookupResult& result = results->ElementAt(i); + + // We will complete partial matches and matches that are stale. + if (!result.Confirmed()) { + nsCOMPtr<nsIUrlClassifierHashCompleter> completer; + nsCString gethashUrl; + nsresult rv; + nsCOMPtr<nsIUrlListManager> listManager = do_GetService( + "@mozilla.org/url-classifier/listmanager;1", &rv); + NS_ENSURE_SUCCESS(rv, rv); + rv = listManager->GetGethashUrl(result.mTableName, gethashUrl); + NS_ENSURE_SUCCESS(rv, rv); + LOG(("The match from %s needs to be completed at %s", + result.mTableName.get(), gethashUrl.get())); + // gethashUrls may be empty in 2 cases: test tables, and on startup where + // we may have found a prefix in an existing table before the listmanager + // has registered the table. In the second case we should not call + // complete. + if ((!gethashUrl.IsEmpty() || + StringBeginsWith(result.mTableName, NS_LITERAL_CSTRING("test-"))) && + mDBService->GetCompleter(result.mTableName, + getter_AddRefs(completer))) { + nsAutoCString partialHash; + partialHash.Assign(reinterpret_cast<char*>(&result.hash.prefix), + PREFIX_SIZE); + + nsresult rv = completer->Complete(partialHash, gethashUrl, this); + if (NS_SUCCEEDED(rv)) { + mPendingCompletions++; + } + } else { + // For tables with no hash completer, a complete hash match is + // good enough, we'll consider it fresh, even if it hasn't been updated + // in 45 minutes. + if (result.Complete()) { + result.mFresh = true; + LOG(("Skipping completion in a table without a valid completer (%s).", + result.mTableName.get())); + } else { + NS_WARNING("Partial match in a table without a valid completer, ignoring partial match."); + } + } + } + } + + LOG(("nsUrlClassifierLookupCallback::LookupComplete [%p] " + "%u pending completions", this, mPendingCompletions)); + if (mPendingCompletions == 0) { + // All results were complete, we're ready! + HandleResults(); + } + + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierLookupCallback::CompletionFinished(nsresult status) +{ + if (LOG_ENABLED()) { + nsAutoCString errorName; + mozilla::GetErrorName(status, errorName); + LOG(("nsUrlClassifierLookupCallback::CompletionFinished [%p, %s]", + this, errorName.get())); + } + + mPendingCompletions--; + if (mPendingCompletions == 0) { + HandleResults(); + } + + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierLookupCallback::Completion(const nsACString& completeHash, + const nsACString& tableName, + uint32_t chunkId) +{ + LOG(("nsUrlClassifierLookupCallback::Completion [%p, %s, %d]", + this, PromiseFlatCString(tableName).get(), chunkId)); + mozilla::safebrowsing::Completion hash; + hash.Assign(completeHash); + + // Send this completion to the store for caching. + if (!mCacheResults) { + mCacheResults = new CacheResultArray(); + if (!mCacheResults) + return NS_ERROR_OUT_OF_MEMORY; + } + + CacheResult result; + result.entry.addChunk = chunkId; + result.entry.complete = hash; + result.table = tableName; + + // OK if this fails, we just won't cache the item. + mCacheResults->AppendElement(result); + + // Check if this matched any of our results. + for (uint32_t i = 0; i < mResults->Length(); i++) { + LookupResult& result = mResults->ElementAt(i); + + // Now, see if it verifies a lookup + if (!result.mNoise + && result.CompleteHash() == hash + && result.mTableName.Equals(tableName)) { + result.mProtocolConfirmed = true; + } + } + + return NS_OK; +} + +nsresult +nsUrlClassifierLookupCallback::HandleResults() +{ + if (!mResults) { + // No results, this URI is clean. + LOG(("nsUrlClassifierLookupCallback::HandleResults [%p, no results]", this)); + return mCallback->HandleEvent(NS_LITERAL_CSTRING("")); + } + MOZ_ASSERT(mPendingCompletions == 0, "HandleResults() should never be " + "called while there are pending completions"); + + LOG(("nsUrlClassifierLookupCallback::HandleResults [%p, %u results]", + this, mResults->Length())); + + nsTArray<nsCString> tables; + // Build a stringified list of result tables. + for (uint32_t i = 0; i < mResults->Length(); i++) { + LookupResult& result = mResults->ElementAt(i); + + // Leave out results that weren't confirmed, as their existence on + // the list can't be verified. Also leave out randomly-generated + // noise. + if (result.mNoise) { + LOG(("Skipping result %X from table %s (noise)", + result.hash.prefix.ToUint32(), result.mTableName.get())); + continue; + } else if (!result.Confirmed()) { + LOG(("Skipping result %X from table %s (not confirmed)", + result.hash.prefix.ToUint32(), result.mTableName.get())); + continue; + } + + LOG(("Confirmed result %X from table %s", + result.hash.prefix.ToUint32(), result.mTableName.get())); + + if (tables.IndexOf(result.mTableName) == nsTArray<nsCString>::NoIndex) { + tables.AppendElement(result.mTableName); + } + } + + // Some parts of this gethash request generated no hits at all. + // Prefixes must have been removed from the database since our last update. + // Save the prefixes we checked to prevent repeated requests + // until the next update. + nsAutoPtr<PrefixArray> cacheMisses(new PrefixArray()); + if (cacheMisses) { + for (uint32_t i = 0; i < mResults->Length(); i++) { + LookupResult &result = mResults->ElementAt(i); + if (!result.Confirmed() && !result.mNoise) { + cacheMisses->AppendElement(result.PrefixHash()); + } + } + // Hands ownership of the miss array back to the worker thread. + mDBService->CacheMisses(cacheMisses.forget()); + } + + if (mCacheResults) { + // This hands ownership of the cache results array back to the worker + // thread. + mDBService->CacheCompletions(mCacheResults.forget()); + } + + nsAutoCString tableStr; + for (uint32_t i = 0; i < tables.Length(); i++) { + if (i != 0) + tableStr.Append(','); + tableStr.Append(tables[i]); + } + + return mCallback->HandleEvent(tableStr); +} + + +// ------------------------------------------------------------------------- +// Helper class for nsIURIClassifier implementation, translates table names +// to nsIURIClassifier enums. + +class nsUrlClassifierClassifyCallback final : public nsIUrlClassifierCallback +{ +public: + NS_DECL_THREADSAFE_ISUPPORTS + NS_DECL_NSIURLCLASSIFIERCALLBACK + + explicit nsUrlClassifierClassifyCallback(nsIURIClassifierCallback *c) + : mCallback(c) + {} + +private: + ~nsUrlClassifierClassifyCallback() {} + + nsCOMPtr<nsIURIClassifierCallback> mCallback; +}; + +NS_IMPL_ISUPPORTS(nsUrlClassifierClassifyCallback, + nsIUrlClassifierCallback) + +NS_IMETHODIMP +nsUrlClassifierClassifyCallback::HandleEvent(const nsACString& tables) +{ + nsresult response = TablesToResponse(tables); + mCallback->OnClassifyComplete(response); + return NS_OK; +} + + +// ------------------------------------------------------------------------- +// Proxy class implementation + +NS_IMPL_ISUPPORTS(nsUrlClassifierDBService, + nsIUrlClassifierDBService, + nsIURIClassifier, + nsIObserver) + +/* static */ nsUrlClassifierDBService* +nsUrlClassifierDBService::GetInstance(nsresult *result) +{ + *result = NS_OK; + if (!sUrlClassifierDBService) { + sUrlClassifierDBService = new nsUrlClassifierDBService(); + if (!sUrlClassifierDBService) { + *result = NS_ERROR_OUT_OF_MEMORY; + return nullptr; + } + + NS_ADDREF(sUrlClassifierDBService); // addref the global + + *result = sUrlClassifierDBService->Init(); + if (NS_FAILED(*result)) { + NS_RELEASE(sUrlClassifierDBService); + return nullptr; + } + } else { + // Already exists, just add a ref + NS_ADDREF(sUrlClassifierDBService); // addref the return result + } + return sUrlClassifierDBService; +} + + +nsUrlClassifierDBService::nsUrlClassifierDBService() + : mCheckMalware(CHECK_MALWARE_DEFAULT) + , mCheckPhishing(CHECK_PHISHING_DEFAULT) + , mCheckTracking(CHECK_TRACKING_DEFAULT) + , mCheckBlockedURIs(CHECK_BLOCKED_DEFAULT) + , mInUpdate(false) +{ +} + +nsUrlClassifierDBService::~nsUrlClassifierDBService() +{ + sUrlClassifierDBService = nullptr; +} + +nsresult +nsUrlClassifierDBService::ReadTablesFromPrefs() +{ + nsCString allTables; + nsCString tables; + Preferences::GetCString(PHISH_TABLE_PREF, &allTables); + + Preferences::GetCString(MALWARE_TABLE_PREF, &tables); + if (!tables.IsEmpty()) { + allTables.Append(','); + allTables.Append(tables); + } + + Preferences::GetCString(DOWNLOAD_BLOCK_TABLE_PREF, &tables); + if (!tables.IsEmpty()) { + allTables.Append(','); + allTables.Append(tables); + } + + Preferences::GetCString(DOWNLOAD_ALLOW_TABLE_PREF, &tables); + if (!tables.IsEmpty()) { + allTables.Append(','); + allTables.Append(tables); + } + + Preferences::GetCString(TRACKING_TABLE_PREF, &tables); + if (!tables.IsEmpty()) { + allTables.Append(','); + allTables.Append(tables); + } + + Preferences::GetCString(TRACKING_WHITELIST_TABLE_PREF, &tables); + if (!tables.IsEmpty()) { + allTables.Append(','); + allTables.Append(tables); + } + + Preferences::GetCString(BLOCKED_TABLE_PREF, &tables); + if (!tables.IsEmpty()) { + allTables.Append(','); + allTables.Append(tables); + } + + Classifier::SplitTables(allTables, mGethashTables); + + Preferences::GetCString(DISALLOW_COMPLETION_TABLE_PREF, &tables); + Classifier::SplitTables(tables, mDisallowCompletionsTables); + + return NS_OK; +} + +nsresult +nsUrlClassifierDBService::Init() +{ + MOZ_ASSERT(NS_IsMainThread(), "Must initialize DB service on main thread"); + nsCOMPtr<nsIXULRuntime> appInfo = do_GetService("@mozilla.org/xre/app-info;1"); + if (appInfo) { + bool inSafeMode = false; + appInfo->GetInSafeMode(&inSafeMode); + if (inSafeMode) { + return NS_ERROR_NOT_AVAILABLE; + } + } + + // Retrieve all the preferences. + mCheckMalware = Preferences::GetBool(CHECK_MALWARE_PREF, + CHECK_MALWARE_DEFAULT); + mCheckPhishing = Preferences::GetBool(CHECK_PHISHING_PREF, + CHECK_PHISHING_DEFAULT); + mCheckTracking = + Preferences::GetBool(CHECK_TRACKING_PREF, CHECK_TRACKING_DEFAULT) || + Preferences::GetBool(CHECK_TRACKING_PB_PREF, CHECK_TRACKING_PB_DEFAULT); + mCheckBlockedURIs = Preferences::GetBool(CHECK_BLOCKED_PREF, + CHECK_BLOCKED_DEFAULT); + uint32_t gethashNoise = Preferences::GetUint(GETHASH_NOISE_PREF, + GETHASH_NOISE_DEFAULT); + gFreshnessGuarantee = Preferences::GetInt(CONFIRM_AGE_PREF, + CONFIRM_AGE_DEFAULT_SEC); + ReadTablesFromPrefs(); + + nsresult rv; + + { + // Force PSM loading on main thread + nsCOMPtr<nsICryptoHash> dummy = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv); + NS_ENSURE_SUCCESS(rv, rv); + } + + { + // Force nsIUrlClassifierUtils loading on main thread. + nsCOMPtr<nsIUrlClassifierUtils> dummy = + do_GetService(NS_URLCLASSIFIERUTILS_CONTRACTID, &rv); + NS_ENSURE_SUCCESS(rv, rv); + } + + // Directory providers must also be accessed on the main thread. + nsCOMPtr<nsIFile> cacheDir; + rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_LOCAL_50_DIR, + getter_AddRefs(cacheDir)); + if (NS_FAILED(rv)) { + rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR, + getter_AddRefs(cacheDir)); + if (NS_FAILED(rv)) { + return rv; + } + } + + // Start the background thread. + rv = NS_NewNamedThread("URL Classifier", &gDbBackgroundThread); + if (NS_FAILED(rv)) + return rv; + + mWorker = new nsUrlClassifierDBServiceWorker(); + if (!mWorker) + return NS_ERROR_OUT_OF_MEMORY; + + rv = mWorker->Init(gethashNoise, cacheDir); + if (NS_FAILED(rv)) { + mWorker = nullptr; + return rv; + } + + // Proxy for calling the worker on the background thread + mWorkerProxy = new UrlClassifierDBServiceWorkerProxy(mWorker); + rv = mWorkerProxy->OpenDb(); + if (NS_FAILED(rv)) { + return rv; + } + + // Add an observer for shutdown + nsCOMPtr<nsIObserverService> observerService = + mozilla::services::GetObserverService(); + if (!observerService) + return NS_ERROR_FAILURE; + + // The application is about to quit + observerService->AddObserver(this, "quit-application", false); + observerService->AddObserver(this, "profile-before-change", false); + + // XXX: Do we *really* need to be able to change all of these at runtime? + // Note: These observers should only be added when everything else above has + // succeeded. Failing to do so can cause long shutdown times in certain + // situations. See Bug 1247798 and Bug 1244803. + Preferences::AddStrongObserver(this, CHECK_MALWARE_PREF); + Preferences::AddStrongObserver(this, CHECK_PHISHING_PREF); + Preferences::AddStrongObserver(this, CHECK_TRACKING_PREF); + Preferences::AddStrongObserver(this, CHECK_TRACKING_PB_PREF); + Preferences::AddStrongObserver(this, CHECK_BLOCKED_PREF); + Preferences::AddStrongObserver(this, GETHASH_NOISE_PREF); + Preferences::AddStrongObserver(this, CONFIRM_AGE_PREF); + Preferences::AddStrongObserver(this, PHISH_TABLE_PREF); + Preferences::AddStrongObserver(this, MALWARE_TABLE_PREF); + Preferences::AddStrongObserver(this, TRACKING_TABLE_PREF); + Preferences::AddStrongObserver(this, TRACKING_WHITELIST_TABLE_PREF); + Preferences::AddStrongObserver(this, BLOCKED_TABLE_PREF); + Preferences::AddStrongObserver(this, DOWNLOAD_BLOCK_TABLE_PREF); + Preferences::AddStrongObserver(this, DOWNLOAD_ALLOW_TABLE_PREF); + Preferences::AddStrongObserver(this, DISALLOW_COMPLETION_TABLE_PREF); + + return NS_OK; +} + +void +nsUrlClassifierDBService::BuildTables(bool aTrackingProtectionEnabled, + nsCString &tables) +{ + nsAutoCString malware; + // LookupURI takes a comma-separated list already. + Preferences::GetCString(MALWARE_TABLE_PREF, &malware); + if (mCheckMalware && !malware.IsEmpty()) { + tables.Append(malware); + } + nsAutoCString phishing; + Preferences::GetCString(PHISH_TABLE_PREF, &phishing); + if (mCheckPhishing && !phishing.IsEmpty()) { + tables.Append(','); + tables.Append(phishing); + } + if (aTrackingProtectionEnabled) { + nsAutoCString tracking, trackingWhitelist; + Preferences::GetCString(TRACKING_TABLE_PREF, &tracking); + if (!tracking.IsEmpty()) { + tables.Append(','); + tables.Append(tracking); + } + Preferences::GetCString(TRACKING_WHITELIST_TABLE_PREF, &trackingWhitelist); + if (!trackingWhitelist.IsEmpty()) { + tables.Append(','); + tables.Append(trackingWhitelist); + } + } + nsAutoCString blocked; + Preferences::GetCString(BLOCKED_TABLE_PREF, &blocked); + if (mCheckBlockedURIs && !blocked.IsEmpty()) { + tables.Append(','); + tables.Append(blocked); + } + + if (StringBeginsWith(tables, NS_LITERAL_CSTRING(","))) { + tables.Cut(0, 1); + } +} + +// nsChannelClassifier is the only consumer of this interface. +NS_IMETHODIMP +nsUrlClassifierDBService::Classify(nsIPrincipal* aPrincipal, + bool aTrackingProtectionEnabled, + nsIURIClassifierCallback* c, + bool* result) +{ + NS_ENSURE_ARG(aPrincipal); + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + if (!(mCheckMalware || mCheckPhishing || aTrackingProtectionEnabled || + mCheckBlockedURIs)) { + *result = false; + return NS_OK; + } + + RefPtr<nsUrlClassifierClassifyCallback> callback = + new nsUrlClassifierClassifyCallback(c); + if (!callback) return NS_ERROR_OUT_OF_MEMORY; + + nsAutoCString tables; + BuildTables(aTrackingProtectionEnabled, tables); + + nsresult rv = LookupURI(aPrincipal, tables, callback, false, result); + if (rv == NS_ERROR_MALFORMED_URI) { + *result = false; + // The URI had no hostname, don't try to classify it. + return NS_OK; + } + NS_ENSURE_SUCCESS(rv, rv); + + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierDBService::ClassifyLocalWithTables(nsIURI *aURI, + const nsACString & aTables, + nsACString & aTableResults) +{ + if (gShuttingDownThread) { + return NS_ERROR_ABORT; + } + + PROFILER_LABEL_FUNC(js::ProfileEntry::Category::OTHER); + MOZ_ASSERT(NS_IsMainThread(), "ClassifyLocalWithTables must be on main thread"); + + nsCOMPtr<nsIURI> uri = NS_GetInnermostURI(aURI); + NS_ENSURE_TRUE(uri, NS_ERROR_FAILURE); + + nsAutoCString key; + // Canonicalize the url + nsCOMPtr<nsIUrlClassifierUtils> utilsService = + do_GetService(NS_URLCLASSIFIERUTILS_CONTRACTID); + nsresult rv = utilsService->GetKeyForURI(uri, key); + NS_ENSURE_SUCCESS(rv, rv); + + nsAutoPtr<LookupResultArray> results(new LookupResultArray()); + if (!results) { + return NS_ERROR_OUT_OF_MEMORY; + } + + // In unittests, we may not have been initalized, so don't crash. + rv = mWorkerProxy->DoLocalLookup(key, aTables, results); + if (NS_SUCCEEDED(rv)) { + aTableResults = ProcessLookupResults(results); + } + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierDBService::Lookup(nsIPrincipal* aPrincipal, + const nsACString& tables, + nsIUrlClassifierCallback* c) +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + bool dummy; + return LookupURI(aPrincipal, tables, c, true, &dummy); +} + +nsresult +nsUrlClassifierDBService::LookupURI(nsIPrincipal* aPrincipal, + const nsACString& tables, + nsIUrlClassifierCallback* c, + bool forceLookup, + bool *didLookup) +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + NS_ENSURE_ARG(aPrincipal); + + if (nsContentUtils::IsSystemPrincipal(aPrincipal)) { + *didLookup = false; + return NS_OK; + } + + if (gShuttingDownThread) { + *didLookup = false; + return NS_ERROR_ABORT; + } + + nsCOMPtr<nsIURI> uri; + nsresult rv = aPrincipal->GetURI(getter_AddRefs(uri)); + NS_ENSURE_SUCCESS(rv, rv); + NS_ENSURE_TRUE(uri, NS_ERROR_FAILURE); + + uri = NS_GetInnermostURI(uri); + NS_ENSURE_TRUE(uri, NS_ERROR_FAILURE); + + nsAutoCString key; + // Canonicalize the url + nsCOMPtr<nsIUrlClassifierUtils> utilsService = + do_GetService(NS_URLCLASSIFIERUTILS_CONTRACTID); + rv = utilsService->GetKeyForURI(uri, key); + if (NS_FAILED(rv)) + return rv; + + if (forceLookup) { + *didLookup = true; + } else { + bool clean = false; + + if (!clean) { + nsCOMPtr<nsIPermissionManager> permissionManager = + services::GetPermissionManager(); + + if (permissionManager) { + uint32_t perm; + rv = permissionManager->TestPermissionFromPrincipal(aPrincipal, + "safe-browsing", &perm); + NS_ENSURE_SUCCESS(rv, rv); + + clean |= (perm == nsIPermissionManager::ALLOW_ACTION); + } + } + + *didLookup = !clean; + if (clean) { + return NS_OK; + } + } + + // Create an nsUrlClassifierLookupCallback object. This object will + // take care of confirming partial hash matches if necessary before + // calling the client's callback. + nsCOMPtr<nsIUrlClassifierLookupCallback> callback = + new nsUrlClassifierLookupCallback(this, c); + if (!callback) + return NS_ERROR_OUT_OF_MEMORY; + + nsCOMPtr<nsIUrlClassifierLookupCallback> proxyCallback = + new UrlClassifierLookupCallbackProxy(callback); + + // Queue this lookup and call the lookup function to flush the queue if + // necessary. + rv = mWorker->QueueLookup(key, tables, proxyCallback); + NS_ENSURE_SUCCESS(rv, rv); + + // This seems to just call HandlePendingLookups. + nsAutoCString dummy; + return mWorkerProxy->Lookup(nullptr, dummy, nullptr); +} + +NS_IMETHODIMP +nsUrlClassifierDBService::GetTables(nsIUrlClassifierCallback* c) +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + // The proxy callback uses the current thread. + nsCOMPtr<nsIUrlClassifierCallback> proxyCallback = + new UrlClassifierCallbackProxy(c); + + return mWorkerProxy->GetTables(proxyCallback); +} + +NS_IMETHODIMP +nsUrlClassifierDBService::SetHashCompleter(const nsACString &tableName, + nsIUrlClassifierHashCompleter *completer) +{ + if (completer) { + mCompleters.Put(tableName, completer); + } else { + mCompleters.Remove(tableName); + } + ClearLastResults(); + return NS_OK; +} + +NS_IMETHODIMP +nsUrlClassifierDBService::SetLastUpdateTime(const nsACString &tableName, + uint64_t lastUpdateTime) +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + return mWorkerProxy->SetLastUpdateTime(tableName, lastUpdateTime); +} + +NS_IMETHODIMP +nsUrlClassifierDBService::ClearLastResults() +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + return mWorkerProxy->ClearLastResults(); +} + +NS_IMETHODIMP +nsUrlClassifierDBService::BeginUpdate(nsIUrlClassifierUpdateObserver *observer, + const nsACString &updateTables) +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + if (mInUpdate) { + LOG(("Already updating, not available")); + return NS_ERROR_NOT_AVAILABLE; + } + + mInUpdate = true; + + // The proxy observer uses the current thread + nsCOMPtr<nsIUrlClassifierUpdateObserver> proxyObserver = + new UrlClassifierUpdateObserverProxy(observer); + + return mWorkerProxy->BeginUpdate(proxyObserver, updateTables); +} + +NS_IMETHODIMP +nsUrlClassifierDBService::BeginStream(const nsACString &table) +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + return mWorkerProxy->BeginStream(table); +} + +NS_IMETHODIMP +nsUrlClassifierDBService::UpdateStream(const nsACString& aUpdateChunk) +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + return mWorkerProxy->UpdateStream(aUpdateChunk); +} + +NS_IMETHODIMP +nsUrlClassifierDBService::FinishStream() +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + return mWorkerProxy->FinishStream(); +} + +NS_IMETHODIMP +nsUrlClassifierDBService::FinishUpdate() +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + mInUpdate = false; + + return mWorkerProxy->FinishUpdate(); +} + + +NS_IMETHODIMP +nsUrlClassifierDBService::CancelUpdate() +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + mInUpdate = false; + + return mWorkerProxy->CancelUpdate(); +} + +NS_IMETHODIMP +nsUrlClassifierDBService::ResetDatabase() +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + return mWorkerProxy->ResetDatabase(); +} + +NS_IMETHODIMP +nsUrlClassifierDBService::ReloadDatabase() +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + return mWorkerProxy->ReloadDatabase(); +} + +nsresult +nsUrlClassifierDBService::CacheCompletions(CacheResultArray *results) +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + return mWorkerProxy->CacheCompletions(results); +} + +nsresult +nsUrlClassifierDBService::CacheMisses(PrefixArray *results) +{ + NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED); + + return mWorkerProxy->CacheMisses(results); +} + +bool +nsUrlClassifierDBService::GetCompleter(const nsACString &tableName, + nsIUrlClassifierHashCompleter **completer) +{ + // If we have specified a completer, go ahead and query it. This is only + // used by tests. + if (mCompleters.Get(tableName, completer)) { + return true; + } + + // If we don't know about this table at all, or are disallowing completions + // for it, skip completion checks. + if (!mGethashTables.Contains(tableName) || + mDisallowCompletionsTables.Contains(tableName)) { + return false; + } + + // Otherwise, call gethash to find the hash completions. + return NS_SUCCEEDED(CallGetService(NS_URLCLASSIFIERHASHCOMPLETER_CONTRACTID, + completer)); +} + +NS_IMETHODIMP +nsUrlClassifierDBService::Observe(nsISupports *aSubject, const char *aTopic, + const char16_t *aData) +{ + if (!strcmp(aTopic, NS_PREFBRANCH_PREFCHANGE_TOPIC_ID)) { + nsresult rv; + nsCOMPtr<nsIPrefBranch> prefs(do_QueryInterface(aSubject, &rv)); + NS_ENSURE_SUCCESS(rv, rv); + Unused << prefs; + + if (NS_LITERAL_STRING(CHECK_MALWARE_PREF).Equals(aData)) { + mCheckMalware = Preferences::GetBool(CHECK_MALWARE_PREF, + CHECK_MALWARE_DEFAULT); + } else if (NS_LITERAL_STRING(CHECK_PHISHING_PREF).Equals(aData)) { + mCheckPhishing = Preferences::GetBool(CHECK_PHISHING_PREF, + CHECK_PHISHING_DEFAULT); + } else if (NS_LITERAL_STRING(CHECK_TRACKING_PREF).Equals(aData) || + NS_LITERAL_STRING(CHECK_TRACKING_PB_PREF).Equals(aData)) { + mCheckTracking = + Preferences::GetBool(CHECK_TRACKING_PREF, CHECK_TRACKING_DEFAULT) || + Preferences::GetBool(CHECK_TRACKING_PB_PREF, CHECK_TRACKING_PB_DEFAULT); + } else if (NS_LITERAL_STRING(CHECK_BLOCKED_PREF).Equals(aData)) { + mCheckBlockedURIs = Preferences::GetBool(CHECK_BLOCKED_PREF, + CHECK_BLOCKED_DEFAULT); + } else if ( + NS_LITERAL_STRING(PHISH_TABLE_PREF).Equals(aData) || + NS_LITERAL_STRING(MALWARE_TABLE_PREF).Equals(aData) || + NS_LITERAL_STRING(TRACKING_TABLE_PREF).Equals(aData) || + NS_LITERAL_STRING(TRACKING_WHITELIST_TABLE_PREF).Equals(aData) || + NS_LITERAL_STRING(BLOCKED_TABLE_PREF).Equals(aData) || + NS_LITERAL_STRING(DOWNLOAD_BLOCK_TABLE_PREF).Equals(aData) || + NS_LITERAL_STRING(DOWNLOAD_ALLOW_TABLE_PREF).Equals(aData) || + NS_LITERAL_STRING(DISALLOW_COMPLETION_TABLE_PREF).Equals(aData)) { + // Just read everything again. + ReadTablesFromPrefs(); + } else if (NS_LITERAL_STRING(CONFIRM_AGE_PREF).Equals(aData)) { + gFreshnessGuarantee = Preferences::GetInt(CONFIRM_AGE_PREF, + CONFIRM_AGE_DEFAULT_SEC); + } + } else if (!strcmp(aTopic, "quit-application")) { + Shutdown(); + } else if (!strcmp(aTopic, "profile-before-change")) { + // Unit test does not receive "quit-application", + // need call shutdown in this case + Shutdown(); + LOG(("joining background thread")); + mWorkerProxy = nullptr; + + if (!gDbBackgroundThread) { + return NS_OK; + } + + nsIThread *backgroundThread = gDbBackgroundThread; + gDbBackgroundThread = nullptr; + backgroundThread->Shutdown(); + NS_RELEASE(backgroundThread); + } else { + return NS_ERROR_UNEXPECTED; + } + + return NS_OK; +} + +// Join the background thread if it exists. +nsresult +nsUrlClassifierDBService::Shutdown() +{ + LOG(("shutting down db service\n")); + + if (!gDbBackgroundThread || gShuttingDownThread) { + return NS_OK; + } + + gShuttingDownThread = true; + + Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_SHUTDOWN_TIME> timer; + + mCompleters.Clear(); + + nsCOMPtr<nsIPrefBranch> prefs = do_GetService(NS_PREFSERVICE_CONTRACTID); + if (prefs) { + prefs->RemoveObserver(CHECK_MALWARE_PREF, this); + prefs->RemoveObserver(CHECK_PHISHING_PREF, this); + prefs->RemoveObserver(CHECK_TRACKING_PREF, this); + prefs->RemoveObserver(CHECK_TRACKING_PB_PREF, this); + prefs->RemoveObserver(CHECK_BLOCKED_PREF, this); + prefs->RemoveObserver(PHISH_TABLE_PREF, this); + prefs->RemoveObserver(MALWARE_TABLE_PREF, this); + prefs->RemoveObserver(TRACKING_TABLE_PREF, this); + prefs->RemoveObserver(TRACKING_WHITELIST_TABLE_PREF, this); + prefs->RemoveObserver(BLOCKED_TABLE_PREF, this); + prefs->RemoveObserver(DOWNLOAD_BLOCK_TABLE_PREF, this); + prefs->RemoveObserver(DOWNLOAD_ALLOW_TABLE_PREF, this); + prefs->RemoveObserver(DISALLOW_COMPLETION_TABLE_PREF, this); + prefs->RemoveObserver(CONFIRM_AGE_PREF, this); + } + + DebugOnly<nsresult> rv; + // First close the db connection. + if (mWorker) { + rv = mWorkerProxy->CancelUpdate(); + NS_ASSERTION(NS_SUCCEEDED(rv), "failed to post cancel update event"); + + rv = mWorkerProxy->CloseDb(); + NS_ASSERTION(NS_SUCCEEDED(rv), "failed to post close db event"); + } + return NS_OK; +} + +nsIThread* +nsUrlClassifierDBService::BackgroundThread() +{ + return gDbBackgroundThread; +} + +// static +bool +nsUrlClassifierDBService::ShutdownHasStarted() +{ + return gShuttingDownThread; +} |