Add m-esr52 at 52.6.0

1 files changed, 1866 insertions, 0 deletions

diff --git a/toolkit/components/url-classifier/nsUrlClassifierDBService.cpp b/toolkit/components/url-classifier/nsUrlClassifierDBService.cpp
new file mode 100644
index 000000000..2ad8b6b51
--- /dev/null
+++ b/toolkit/components/url-classifier/nsUrlClassifierDBService.cpp
@@ -0,0 +1,1866 @@
+//* -*- Mode: C++; tab-width: 8; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+#include "nsAutoPtr.h"
+#include "nsCOMPtr.h"
+#include "nsAppDirectoryServiceDefs.h"
+#include "nsCRT.h"
+#include "nsICryptoHash.h"
+#include "nsICryptoHMAC.h"
+#include "nsIDirectoryService.h"
+#include "nsIKeyModule.h"
+#include "nsIObserverService.h"
+#include "nsIPermissionManager.h"
+#include "nsIPrefBranch.h"
+#include "nsIPrefService.h"
+#include "nsIProperties.h"
+#include "nsToolkitCompsCID.h"
+#include "nsIUrlClassifierUtils.h"
+#include "nsIXULRuntime.h"
+#include "nsUrlClassifierDBService.h"
+#include "nsUrlClassifierUtils.h"
+#include "nsUrlClassifierProxies.h"
+#include "nsURILoader.h"
+#include "nsString.h"
+#include "nsReadableUtils.h"
+#include "nsTArray.h"
+#include "nsNetCID.h"
+#include "nsThreadUtils.h"
+#include "nsXPCOMStrings.h"
+#include "nsProxyRelease.h"
+#include "nsString.h"
+#include "mozilla/Atomics.h"
+#include "mozilla/DebugOnly.h"
+#include "mozilla/ErrorNames.h"
+#include "mozilla/Mutex.h"
+#include "mozilla/Preferences.h"
+#include "mozilla/TimeStamp.h"
+#include "mozilla/Telemetry.h"
+#include "mozilla/Logging.h"
+#include "prprf.h"
+#include "prnetdb.h"
+#include "Entries.h"
+#include "HashStore.h"
+#include "Classifier.h"
+#include "ProtocolParser.h"
+#include "mozilla/Attributes.h"
+#include "nsIPrincipal.h"
+#include "Classifier.h"
+#include "ProtocolParser.h"
+#include "nsContentUtils.h"
+
+namespace mozilla {
+namespace safebrowsing {
+
+nsresult
+TablesToResponse(const nsACString& tables)
+{
+  if (tables.IsEmpty()) {
+    return NS_OK;
+  }
+
+  // We don't check mCheckMalware and friends because BuildTables never
+  // includes a table that is not enabled.
+  if (FindInReadable(NS_LITERAL_CSTRING("-malware-"), tables)) {
+    return NS_ERROR_MALWARE_URI;
+  }
+  if (FindInReadable(NS_LITERAL_CSTRING("-phish-"), tables)) {
+    return NS_ERROR_PHISHING_URI;
+  }
+  if (FindInReadable(NS_LITERAL_CSTRING("-unwanted-"), tables)) {
+    return NS_ERROR_UNWANTED_URI;
+  }
+  if (FindInReadable(NS_LITERAL_CSTRING("-track-"), tables)) {
+    return NS_ERROR_TRACKING_URI;
+  }
+  if (FindInReadable(NS_LITERAL_CSTRING("-block-"), tables)) {
+    return NS_ERROR_BLOCKED_URI;
+  }
+  return NS_OK;
+}
+
+} // namespace safebrowsing
+} // namespace mozilla
+
+using namespace mozilla;
+using namespace mozilla::safebrowsing;
+
+// MOZ_LOG=UrlClassifierDbService:5
+LazyLogModule gUrlClassifierDbServiceLog("UrlClassifierDbService");
+#define LOG(args) MOZ_LOG(gUrlClassifierDbServiceLog, mozilla::LogLevel::Debug, args)
+#define LOG_ENABLED() MOZ_LOG_TEST(gUrlClassifierDbServiceLog, mozilla::LogLevel::Debug)
+
+// Prefs for implementing nsIURIClassifier to block page loads
+#define CHECK_MALWARE_PREF      "browser.safebrowsing.malware.enabled"
+#define CHECK_MALWARE_DEFAULT   false
+
+#define CHECK_PHISHING_PREF     "browser.safebrowsing.phishing.enabled"
+#define CHECK_PHISHING_DEFAULT  false
+
+#define CHECK_TRACKING_PREF     "privacy.trackingprotection.enabled"
+#define CHECK_TRACKING_DEFAULT  false
+
+#define CHECK_TRACKING_PB_PREF    "privacy.trackingprotection.pbmode.enabled"
+#define CHECK_TRACKING_PB_DEFAULT false
+
+#define CHECK_BLOCKED_PREF    "browser.safebrowsing.blockedURIs.enabled"
+#define CHECK_BLOCKED_DEFAULT false
+
+#define GETHASH_NOISE_PREF      "urlclassifier.gethashnoise"
+#define GETHASH_NOISE_DEFAULT   4
+
+// Comma-separated lists
+#define MALWARE_TABLE_PREF              "urlclassifier.malwareTable"
+#define PHISH_TABLE_PREF                "urlclassifier.phishTable"
+#define TRACKING_TABLE_PREF             "urlclassifier.trackingTable"
+#define TRACKING_WHITELIST_TABLE_PREF   "urlclassifier.trackingWhitelistTable"
+#define BLOCKED_TABLE_PREF              "urlclassifier.blockedTable"
+#define DOWNLOAD_BLOCK_TABLE_PREF       "urlclassifier.downloadBlockTable"
+#define DOWNLOAD_ALLOW_TABLE_PREF       "urlclassifier.downloadAllowTable"
+#define DISALLOW_COMPLETION_TABLE_PREF  "urlclassifier.disallow_completions"
+
+#define CONFIRM_AGE_PREF        "urlclassifier.max-complete-age"
+#define CONFIRM_AGE_DEFAULT_SEC (45 * 60)
+
+class nsUrlClassifierDBServiceWorker;
+
+// Singleton instance.
+static nsUrlClassifierDBService* sUrlClassifierDBService;
+
+nsIThread* nsUrlClassifierDBService::gDbBackgroundThread = nullptr;
+
+// Once we've committed to shutting down, don't do work in the background
+// thread.
+static bool gShuttingDownThread = false;
+
+static mozilla::Atomic<int32_t> gFreshnessGuarantee(CONFIRM_AGE_DEFAULT_SEC);
+
+NS_IMPL_ISUPPORTS(nsUrlClassifierDBServiceWorker,
+                  nsIUrlClassifierDBService)
+
+nsUrlClassifierDBServiceWorker::nsUrlClassifierDBServiceWorker()
+  : mInStream(false)
+  , mGethashNoise(0)
+  , mPendingLookupLock("nsUrlClassifierDBServerWorker.mPendingLookupLock")
+{
+}
+
+nsUrlClassifierDBServiceWorker::~nsUrlClassifierDBServiceWorker()
+{
+  NS_ASSERTION(!mClassifier,
+               "Db connection not closed, leaking memory!  Call CloseDb "
+               "to close the connection.");
+}
+
+nsresult
+nsUrlClassifierDBServiceWorker::Init(uint32_t aGethashNoise,
+                                     nsCOMPtr<nsIFile> aCacheDir)
+{
+  mGethashNoise = aGethashNoise;
+  mCacheDir = aCacheDir;
+
+  ResetUpdate();
+
+  return NS_OK;
+}
+
+nsresult
+nsUrlClassifierDBServiceWorker::QueueLookup(const nsACString& spec,
+                                            const nsACString& tables,
+                                            nsIUrlClassifierLookupCallback* callback)
+{
+  MutexAutoLock lock(mPendingLookupLock);
+
+  PendingLookup* lookup = mPendingLookups.AppendElement();
+  if (!lookup) return NS_ERROR_OUT_OF_MEMORY;
+
+  lookup->mStartTime = TimeStamp::Now();
+  lookup->mKey = spec;
+  lookup->mCallback = callback;
+  lookup->mTables = tables;
+
+  return NS_OK;
+}
+
+nsresult
+nsUrlClassifierDBServiceWorker::DoLocalLookup(const nsACString& spec,
+                                              const nsACString& tables,
+                                              LookupResultArray* results)
+{
+  if (gShuttingDownThread) {
+    return NS_ERROR_ABORT;
+  }
+
+  MOZ_ASSERT(!NS_IsMainThread(), "DoLocalLookup must be on background thread");
+  if (!results) {
+    return NS_ERROR_FAILURE;
+  }
+  // Bail if we haven't been initialized on the background thread.
+  if (!mClassifier) {
+    return NS_ERROR_NOT_AVAILABLE;
+  }
+
+  // We ignore failures from Check because we'd rather return the
+  // results that were found than fail.
+  mClassifier->Check(spec, tables, gFreshnessGuarantee, *results);
+
+  LOG(("Found %d results.", results->Length()));
+  return NS_OK;
+}
+
+static nsCString
+ProcessLookupResults(LookupResultArray* results)
+{
+  // Build a stringified list of result tables.
+  nsTArray<nsCString> tables;
+  for (uint32_t i = 0; i < results->Length(); i++) {
+    LookupResult& result = results->ElementAt(i);
+    MOZ_ASSERT(!result.mNoise, "Lookup results should not have noise added");
+    LOG(("Found result from table %s", result.mTableName.get()));
+    if (tables.IndexOf(result.mTableName) == nsTArray<nsCString>::NoIndex) {
+      tables.AppendElement(result.mTableName);
+    }
+  }
+  nsAutoCString tableStr;
+  for (uint32_t i = 0; i < tables.Length(); i++) {
+    if (i != 0)
+      tableStr.Append(',');
+    tableStr.Append(tables[i]);
+  }
+  return tableStr;
+}
+
+/**
+ * Lookup up a key in the database is a two step process:
+ *
+ * a) First we look for any Entries in the database that might apply to this
+ *    url.  For each URL there are one or two possible domain names to check:
+ *    the two-part domain name (example.com) and the three-part name
+ *    (www.example.com).  We check the database for both of these.
+ * b) If we find any entries, we check the list of fragments for that entry
+ *    against the possible subfragments of the URL as described in the
+ *    "Simplified Regular Expression Lookup" section of the protocol doc.
+ */
+nsresult
+nsUrlClassifierDBServiceWorker::DoLookup(const nsACString& spec,
+                                         const nsACString& tables,
+                                         nsIUrlClassifierLookupCallback* c)
+{
+  if (gShuttingDownThread) {
+    c->LookupComplete(nullptr);
+    return NS_ERROR_NOT_INITIALIZED;
+  }
+
+  PRIntervalTime clockStart = 0;
+  if (LOG_ENABLED()) {
+    clockStart = PR_IntervalNow();
+  }
+
+  nsAutoPtr<LookupResultArray> results(new LookupResultArray());
+  if (!results) {
+    c->LookupComplete(nullptr);
+    return NS_ERROR_OUT_OF_MEMORY;
+  }
+
+  nsresult rv = DoLocalLookup(spec, tables, results);
+  if (NS_FAILED(rv)) {
+    c->LookupComplete(nullptr);
+    return rv;
+  }
+
+  LOG(("Found %d results.", results->Length()));
+
+
+  if (LOG_ENABLED()) {
+    PRIntervalTime clockEnd = PR_IntervalNow();
+    LOG(("query took %dms\n",
+         PR_IntervalToMilliseconds(clockEnd - clockStart)));
+  }
+
+  nsAutoPtr<LookupResultArray> completes(new LookupResultArray());
+
+  for (uint32_t i = 0; i < results->Length(); i++) {
+    if (!mMissCache.Contains(results->ElementAt(i).hash.prefix)) {
+      completes->AppendElement(results->ElementAt(i));
+    }
+  }
+
+  for (uint32_t i = 0; i < completes->Length(); i++) {
+    if (!completes->ElementAt(i).Confirmed()) {
+      // We're going to be doing a gethash request, add some extra entries.
+      // Note that we cannot pass the first two by reference, because we
+      // add to completes, whicah can cause completes to reallocate and move.
+      AddNoise(completes->ElementAt(i).hash.prefix,
+               completes->ElementAt(i).mTableName,
+               mGethashNoise, *completes);
+      break;
+    }
+  }
+
+  // At this point ownership of 'results' is handed to the callback.
+  c->LookupComplete(completes.forget());
+
+  return NS_OK;
+}
+
+nsresult
+nsUrlClassifierDBServiceWorker::HandlePendingLookups()
+{
+  if (gShuttingDownThread) {
+    return NS_ERROR_ABORT;
+  }
+
+  MutexAutoLock lock(mPendingLookupLock);
+  while (mPendingLookups.Length() > 0) {
+    PendingLookup lookup = mPendingLookups[0];
+    mPendingLookups.RemoveElementAt(0);
+    {
+      MutexAutoUnlock unlock(mPendingLookupLock);
+      DoLookup(lookup.mKey, lookup.mTables, lookup.mCallback);
+    }
+    double lookupTime = (TimeStamp::Now() - lookup.mStartTime).ToMilliseconds();
+    Telemetry::Accumulate(Telemetry::URLCLASSIFIER_LOOKUP_TIME,
+                          static_cast<uint32_t>(lookupTime));
+  }
+
+  return NS_OK;
+}
+
+nsresult
+nsUrlClassifierDBServiceWorker::AddNoise(const Prefix aPrefix,
+                                         const nsCString tableName,
+                                         uint32_t aCount,
+                                         LookupResultArray& results)
+{
+  if (gShuttingDownThread) {
+    return NS_ERROR_ABORT;
+  }
+
+  if (aCount < 1) {
+    return NS_OK;
+  }
+
+  PrefixArray noiseEntries;
+  nsresult rv = mClassifier->ReadNoiseEntries(aPrefix, tableName,
+                                              aCount, &noiseEntries);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  for (uint32_t i = 0; i < noiseEntries.Length(); i++) {
+    LookupResult *result = results.AppendElement();
+    if (!result)
+      return NS_ERROR_OUT_OF_MEMORY;
+
+    result->hash.prefix = noiseEntries[i];
+    result->mNoise = true;
+
+    result->mTableName.Assign(tableName);
+  }
+
+  return NS_OK;
+}
+
+// Lookup a key in the db.
+NS_IMETHODIMP
+nsUrlClassifierDBServiceWorker::Lookup(nsIPrincipal* aPrincipal,
+                                       const nsACString& aTables,
+                                       nsIUrlClassifierCallback* c)
+{
+  if (gShuttingDownThread) {
+    return NS_ERROR_ABORT;
+  }
+
+  return HandlePendingLookups();
+}
+
+NS_IMETHODIMP
+nsUrlClassifierDBServiceWorker::GetTables(nsIUrlClassifierCallback* c)
+{
+  if (gShuttingDownThread) {
+    return NS_ERROR_NOT_INITIALIZED;
+  }
+
+  nsresult rv = OpenDb();
+  if (NS_FAILED(rv)) {
+    NS_ERROR("Unable to open SafeBrowsing database");
+    return NS_ERROR_FAILURE;
+  }
+
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  nsAutoCString response;
+  mClassifier->TableRequest(response);
+  LOG(("GetTables: %s", response.get()));
+  c->HandleEvent(response);
+
+  return rv;
+}
+
+void
+nsUrlClassifierDBServiceWorker::ResetStream()
+{
+  LOG(("ResetStream"));
+  mInStream = false;
+  mProtocolParser = nullptr;
+}
+
+void
+nsUrlClassifierDBServiceWorker::ResetUpdate()
+{
+  LOG(("ResetUpdate"));
+  mUpdateWaitSec = 0;
+  mUpdateStatus = NS_OK;
+  mUpdateObserver = nullptr;
+}
+
+NS_IMETHODIMP
+nsUrlClassifierDBServiceWorker::SetHashCompleter(const nsACString &tableName,
+                                                 nsIUrlClassifierHashCompleter *completer)
+{
+  return NS_ERROR_NOT_IMPLEMENTED;
+}
+
+NS_IMETHODIMP
+nsUrlClassifierDBServiceWorker::BeginUpdate(nsIUrlClassifierUpdateObserver *observer,
+                                            const nsACString &tables)
+{
+  LOG(("nsUrlClassifierDBServiceWorker::BeginUpdate [%s]", PromiseFlatCString(tables).get()));
+
+  if (gShuttingDownThread) {
+    return NS_ERROR_NOT_INITIALIZED;
+  }
+
+  NS_ENSURE_STATE(!mUpdateObserver);
+
+  nsresult rv = OpenDb();
+  if (NS_FAILED(rv)) {
+    NS_ERROR("Unable to open SafeBrowsing database");
+    return NS_ERROR_FAILURE;
+  }
+
+  mUpdateStatus = NS_OK;
+  mUpdateObserver = observer;
+  Classifier::SplitTables(tables, mUpdateTables);
+
+  return NS_OK;
+}
+
+// Called from the stream updater.
+NS_IMETHODIMP
+nsUrlClassifierDBServiceWorker::BeginStream(const nsACString &table)
+{
+  LOG(("nsUrlClassifierDBServiceWorker::BeginStream"));
+  MOZ_ASSERT(!NS_IsMainThread(), "Streaming must be on the background thread");
+
+  if (gShuttingDownThread) {
+    return NS_ERROR_NOT_INITIALIZED;
+  }
+
+  NS_ENSURE_STATE(mUpdateObserver);
+  NS_ENSURE_STATE(!mInStream);
+
+  mInStream = true;
+
+  NS_ASSERTION(!mProtocolParser, "Should not have a protocol parser.");
+
+  // Check if we should use protobuf to parse the update.
+  bool useProtobuf = false;
+  for (size_t i = 0; i < mUpdateTables.Length(); i++) {
+    bool isCurProtobuf =
+      StringEndsWith(mUpdateTables[i], NS_LITERAL_CSTRING("-proto"));
+
+    if (0 == i) {
+      // Use the first table name to decice if all the subsequent tables
+      // should be '-proto'.
+      useProtobuf = isCurProtobuf;
+      continue;
+    }
+
+    if (useProtobuf != isCurProtobuf) {
+      NS_WARNING("Cannot mix 'proto' tables with other types "
+                 "within the same provider.");
+      break;
+    }
+  }
+
+  mProtocolParser = (useProtobuf ? static_cast<ProtocolParser*>(new ProtocolParserProtobuf())
+                                 : static_cast<ProtocolParser*>(new ProtocolParserV2()));
+  if (!mProtocolParser)
+    return NS_ERROR_OUT_OF_MEMORY;
+
+  mProtocolParser->Init(mCryptoHash);
+
+  if (!table.IsEmpty()) {
+    mProtocolParser->SetCurrentTable(table);
+  }
+
+  mProtocolParser->SetRequestedTables(mUpdateTables);
+
+  return NS_OK;
+}
+
+/**
+ * Updating the database:
+ *
+ * The Update() method takes a series of chunks separated with control data,
+ * as described in
+ * http://code.google.com/p/google-safe-browsing/wiki/Protocolv2Spec
+ *
+ * It will iterate through the control data until it reaches a chunk.  By
+ * the time it reaches a chunk, it should have received
+ * a) the table to which this chunk applies
+ * b) the type of chunk (add, delete, expire add, expire delete).
+ * c) the chunk ID
+ * d) the length of the chunk.
+ *
+ * For add and subtract chunks, it needs to read the chunk data (expires
+ * don't have any data).  Chunk data is a list of URI fragments whose
+ * encoding depends on the type of table (which is indicated by the end
+ * of the table name):
+ * a) tables ending with -exp are a zlib-compressed list of URI fragments
+ *    separated by newlines.
+ * b) tables ending with -sha128 have the form
+ *    [domain][N][frag0]...[fragN]
+ *       16    1   16        16
+ *    If N is 0, the domain is reused as a fragment.
+ * c) any other tables are assumed to be a plaintext list of URI fragments
+ *    separated by newlines.
+ *
+ * Update() can be fed partial data;  It will accumulate data until there is
+ * enough to act on.  Finish() should be called when there will be no more
+ * data.
+ */
+NS_IMETHODIMP
+nsUrlClassifierDBServiceWorker::UpdateStream(const nsACString& chunk)
+{
+  if (gShuttingDownThread) {
+    return NS_ERROR_NOT_INITIALIZED;
+  }
+
+  NS_ENSURE_STATE(mInStream);
+
+  HandlePendingLookups();
+
+  // Feed the chunk to the parser.
+  return mProtocolParser->AppendStream(chunk);
+}
+
+NS_IMETHODIMP
+nsUrlClassifierDBServiceWorker::FinishStream()
+{
+  if (gShuttingDownThread) {
+    LOG(("shutting down"));
+    return NS_ERROR_NOT_INITIALIZED;
+  }
+
+  NS_ENSURE_STATE(mInStream);
+  NS_ENSURE_STATE(mUpdateObserver);
+
+  mInStream = false;
+
+  mProtocolParser->End();
+
+  if (NS_SUCCEEDED(mProtocolParser->Status())) {
+    if (mProtocolParser->UpdateWaitSec()) {
+      mUpdateWaitSec = mProtocolParser->UpdateWaitSec();
+    }
+    // XXX: Only allow forwards from the initial update?
+    const nsTArray<ProtocolParser::ForwardedUpdate> &forwards =
+      mProtocolParser->Forwards();
+    for (uint32_t i = 0; i < forwards.Length(); i++) {
+      const ProtocolParser::ForwardedUpdate &forward = forwards[i];
+      mUpdateObserver->UpdateUrlRequested(forward.url, forward.table);
+    }
+    // Hold on to any TableUpdate objects that were created by the
+    // parser.
+    mTableUpdates.AppendElements(mProtocolParser->GetTableUpdates());
+    mProtocolParser->ForgetTableUpdates();
+
+#ifdef MOZ_SAFEBROWSING_DUMP_FAILED_UPDATES
+    // The assignment involves no string copy since the source string is sharable.
+    mRawTableUpdates = mProtocolParser->GetRawTableUpdates();
+#endif
+  } else {
+    LOG(("nsUrlClassifierDBService::FinishStream Failed to parse the stream "
+         "using mProtocolParser."));
+    mUpdateStatus = mProtocolParser->Status();
+  }
+  mUpdateObserver->StreamFinished(mProtocolParser->Status(), 0);
+
+  if (NS_SUCCEEDED(mUpdateStatus)) {
+    if (mProtocolParser->ResetRequested()) {
+      mClassifier->ResetTables(Classifier::Clear_All, mUpdateTables);
+    }
+  }
+
+  mProtocolParser = nullptr;
+
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+nsUrlClassifierDBServiceWorker::FinishUpdate()
+{
+  if (gShuttingDownThread) {
+    return NS_ERROR_NOT_INITIALIZED;
+  }
+
+  NS_ENSURE_STATE(mUpdateObserver);
+
+  if (NS_SUCCEEDED(mUpdateStatus)) {
+    mUpdateStatus = ApplyUpdate();
+  } else {
+    LOG(("nsUrlClassifierDBServiceWorker::FinishUpdate() Not running "
+         "ApplyUpdate() since the update has already failed."));
+  }
+
+  mMissCache.Clear();
+
+  if (NS_SUCCEEDED(mUpdateStatus)) {
+    LOG(("Notifying success: %d", mUpdateWaitSec));
+    mUpdateObserver->UpdateSuccess(mUpdateWaitSec);
+  } else if (NS_ERROR_NOT_IMPLEMENTED == mUpdateStatus) {
+    LOG(("Treating NS_ERROR_NOT_IMPLEMENTED a successful update "
+         "but still mark it spoiled."));
+    mUpdateObserver->UpdateSuccess(0);
+    mClassifier->ResetTables(Classifier::Clear_Cache, mUpdateTables);
+  } else {
+    if (LOG_ENABLED()) {
+      nsAutoCString errorName;
+      mozilla::GetErrorName(mUpdateStatus, errorName);
+      LOG(("Notifying error: %s (%d)", errorName.get(), mUpdateStatus));
+    }
+
+    mUpdateObserver->UpdateError(mUpdateStatus);
+    /*
+     * mark the tables as spoiled(clear cache in LookupCache), we don't want to
+     * block hosts longer than normal because our update failed
+    */
+    mClassifier->ResetTables(Classifier::Clear_Cache, mUpdateTables);
+  }
+  mUpdateObserver = nullptr;
+
+  return NS_OK;
+}
+
+nsresult
+nsUrlClassifierDBServiceWorker::ApplyUpdate()
+{
+  LOG(("nsUrlClassifierDBServiceWorker::ApplyUpdate()"));
+  nsresult rv = mClassifier->ApplyUpdates(&mTableUpdates);
+
+#ifdef MOZ_SAFEBROWSING_DUMP_FAILED_UPDATES
+  if (NS_FAILED(rv) && NS_ERROR_OUT_OF_MEMORY != rv) {
+    mClassifier->DumpRawTableUpdates(mRawTableUpdates);
+  }
+  // Invalidate the raw table updates.
+  mRawTableUpdates = EmptyCString();
+#endif
+
+  return rv;
+}
+
+NS_IMETHODIMP
+nsUrlClassifierDBServiceWorker::ResetDatabase()
+{
+  nsresult rv = OpenDb();
+
+  if (NS_SUCCEEDED(rv)) {
+    mClassifier->Reset();
+  }
+
+  rv = CloseDb();
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+nsUrlClassifierDBServiceWorker::ReloadDatabase()
+{
+  nsTArray<nsCString> tables;
+  nsTArray<int64_t> lastUpdateTimes;
+  nsresult rv = mClassifier->ActiveTables(tables);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  // We need to make sure lastupdatetime is set after reload database
+  // Otherwise request will be skipped if it is not confirmed.
+  for (uint32_t table = 0; table < tables.Length(); table++) {
+    lastUpdateTimes.AppendElement(mClassifier->GetLastUpdateTime(tables[table]));
+  }
+
+  // This will null out mClassifier
+  rv = CloseDb();
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  // Create new mClassifier and load prefixset and completions from disk.
+  rv = OpenDb();
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  for (uint32_t table = 0; table < tables.Length(); table++) {
+    int64_t time = lastUpdateTimes[table];
+    if (time) {
+      mClassifier->SetLastUpdateTime(tables[table], lastUpdateTimes[table]);
+    }
+  }
+
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+nsUrlClassifierDBServiceWorker::CancelUpdate()
+{
+  LOG(("nsUrlClassifierDBServiceWorker::CancelUpdate"));
+
+  if (mUpdateObserver) {
+    LOG(("UpdateObserver exists, cancelling"));
+
+    mUpdateStatus = NS_BINDING_ABORTED;
+
+    mUpdateObserver->UpdateError(mUpdateStatus);
+
+    /*
+     * mark the tables as spoiled(clear cache in LookupCache), we don't want to
+     * block hosts longer than normal because our update failed
+    */
+    mClassifier->ResetTables(Classifier::Clear_Cache, mUpdateTables);
+
+    ResetStream();
+    ResetUpdate();
+  } else {
+    LOG(("No UpdateObserver, nothing to cancel"));
+  }
+
+  return NS_OK;
+}
+
+// Allows the main thread to delete the connection which may be in
+// a background thread.
+// XXX This could be turned into a single shutdown event so the logic
+// is simpler in nsUrlClassifierDBService::Shutdown.
+nsresult
+nsUrlClassifierDBServiceWorker::CloseDb()
+{
+  if (mClassifier) {
+    mClassifier->Close();
+    mClassifier = nullptr;
+  }
+
+  mCryptoHash = nullptr;
+  LOG(("urlclassifier db closed\n"));
+
+  return NS_OK;
+}
+
+nsresult
+nsUrlClassifierDBServiceWorker::CacheCompletions(CacheResultArray *results)
+{
+  if (gShuttingDownThread) {
+    return NS_ERROR_ABORT;
+  }
+
+  LOG(("nsUrlClassifierDBServiceWorker::CacheCompletions [%p]", this));
+  if (!mClassifier)
+    return NS_OK;
+
+  // Ownership is transferred in to us
+  nsAutoPtr<CacheResultArray> resultsPtr(results);
+
+  if (mLastResults == *resultsPtr) {
+    LOG(("Skipping completions that have just been cached already."));
+    return NS_OK;
+  }
+
+  nsAutoPtr<ProtocolParserV2> pParse(new ProtocolParserV2());
+  nsTArray<TableUpdate*> updates;
+
+  // Only cache results for tables that we have, don't take
+  // in tables we might accidentally have hit during a completion.
+  // This happens due to goog vs googpub lists existing.
+  nsTArray<nsCString> tables;
+  nsresult rv = mClassifier->ActiveTables(tables);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  for (uint32_t i = 0; i < resultsPtr->Length(); i++) {
+    bool activeTable = false;
+    for (uint32_t table = 0; table < tables.Length(); table++) {
+      if (tables[table].Equals(resultsPtr->ElementAt(i).table)) {
+        activeTable = true;
+        break;
+      }
+    }
+    if (activeTable) {
+      TableUpdateV2* tuV2 = TableUpdate::Cast<TableUpdateV2>(
+        pParse->GetTableUpdate(resultsPtr->ElementAt(i).table));
+
+      NS_ENSURE_TRUE(tuV2, NS_ERROR_FAILURE);
+
+      LOG(("CacheCompletion Addchunk %d hash %X", resultsPtr->ElementAt(i).entry.addChunk,
+           resultsPtr->ElementAt(i).entry.ToUint32()));
+      rv = tuV2->NewAddComplete(resultsPtr->ElementAt(i).entry.addChunk,
+                                resultsPtr->ElementAt(i).entry.complete);
+      if (NS_FAILED(rv)) {
+        // We can bail without leaking here because ForgetTableUpdates
+        // hasn't been called yet.
+        return rv;
+      }
+      rv = tuV2->NewAddChunk(resultsPtr->ElementAt(i).entry.addChunk);
+      if (NS_FAILED(rv)) {
+        return rv;
+      }
+      updates.AppendElement(tuV2);
+      pParse->ForgetTableUpdates();
+    } else {
+      LOG(("Completion received, but table is not active, so not caching."));
+    }
+   }
+
+  mClassifier->ApplyFullHashes(&updates);
+  mLastResults = *resultsPtr;
+  return NS_OK;
+}
+
+nsresult
+nsUrlClassifierDBServiceWorker::CacheMisses(PrefixArray *results)
+{
+  LOG(("nsUrlClassifierDBServiceWorker::CacheMisses [%p] %d",
+       this, results->Length()));
+
+  // Ownership is transferred in to us
+  nsAutoPtr<PrefixArray> resultsPtr(results);
+
+  for (uint32_t i = 0; i < resultsPtr->Length(); i++) {
+    mMissCache.AppendElement(resultsPtr->ElementAt(i));
+  }
+  return NS_OK;
+}
+
+nsresult
+nsUrlClassifierDBServiceWorker::OpenDb()
+{
+  if (gShuttingDownThread) {
+    return NS_ERROR_ABORT;
+  }
+
+  MOZ_ASSERT(!NS_IsMainThread(), "Must initialize DB on background thread");
+  // Connection already open, don't do anything.
+  if (mClassifier) {
+    return NS_OK;
+  }
+
+  nsresult rv;
+  mCryptoHash = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  nsAutoPtr<Classifier> classifier(new Classifier());
+  if (!classifier) {
+    return NS_ERROR_OUT_OF_MEMORY;
+  }
+
+  rv = classifier->Open(*mCacheDir);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  mClassifier = classifier;
+
+  return NS_OK;
+}
+
+nsresult
+nsUrlClassifierDBServiceWorker::SetLastUpdateTime(const nsACString &table,
+                                                  uint64_t updateTime)
+{
+  MOZ_ASSERT(!NS_IsMainThread(), "Must be on the background thread");
+  MOZ_ASSERT(mClassifier, "Classifier connection must be opened");
+
+  mClassifier->SetLastUpdateTime(table, updateTime);
+
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+nsUrlClassifierDBServiceWorker::ClearLastResults()
+{
+  MOZ_ASSERT(!NS_IsMainThread(), "Must be on the background thread");
+  mLastResults.Clear();
+  return NS_OK;
+}
+
+
+// -------------------------------------------------------------------------
+// nsUrlClassifierLookupCallback
+//
+// This class takes the results of a lookup found on the worker thread
+// and handles any necessary partial hash expansions before calling
+// the client callback.
+
+class nsUrlClassifierLookupCallback final : public nsIUrlClassifierLookupCallback
+                                          , public nsIUrlClassifierHashCompleterCallback
+{
+public:
+  NS_DECL_THREADSAFE_ISUPPORTS
+  NS_DECL_NSIURLCLASSIFIERLOOKUPCALLBACK
+  NS_DECL_NSIURLCLASSIFIERHASHCOMPLETERCALLBACK
+
+  nsUrlClassifierLookupCallback(nsUrlClassifierDBService *dbservice,
+                                nsIUrlClassifierCallback *c)
+    : mDBService(dbservice)
+    , mResults(nullptr)
+    , mPendingCompletions(0)
+    , mCallback(c)
+    {}
+
+private:
+  ~nsUrlClassifierLookupCallback();
+
+  nsresult HandleResults();
+
+  RefPtr<nsUrlClassifierDBService> mDBService;
+  nsAutoPtr<LookupResultArray> mResults;
+
+  // Completed results to send back to the worker for caching.
+  nsAutoPtr<CacheResultArray> mCacheResults;
+
+  uint32_t mPendingCompletions;
+  nsCOMPtr<nsIUrlClassifierCallback> mCallback;
+};
+
+NS_IMPL_ISUPPORTS(nsUrlClassifierLookupCallback,
+                  nsIUrlClassifierLookupCallback,
+                  nsIUrlClassifierHashCompleterCallback)
+
+nsUrlClassifierLookupCallback::~nsUrlClassifierLookupCallback()
+{
+  if (mCallback) {
+    NS_ReleaseOnMainThread(mCallback.forget());
+  }
+}
+
+NS_IMETHODIMP
+nsUrlClassifierLookupCallback::LookupComplete(nsTArray<LookupResult>* results)
+{
+  NS_ASSERTION(mResults == nullptr,
+               "Should only get one set of results per nsUrlClassifierLookupCallback!");
+
+  if (!results) {
+    HandleResults();
+    return NS_OK;
+  }
+
+  mResults = results;
+
+  // Check the results entries that need to be completed.
+  for (uint32_t i = 0; i < results->Length(); i++) {
+    LookupResult& result = results->ElementAt(i);
+
+    // We will complete partial matches and matches that are stale.
+    if (!result.Confirmed()) {
+      nsCOMPtr<nsIUrlClassifierHashCompleter> completer;
+      nsCString gethashUrl;
+      nsresult rv;
+      nsCOMPtr<nsIUrlListManager> listManager = do_GetService(
+        "@mozilla.org/url-classifier/listmanager;1", &rv);
+      NS_ENSURE_SUCCESS(rv, rv);
+      rv = listManager->GetGethashUrl(result.mTableName, gethashUrl);
+      NS_ENSURE_SUCCESS(rv, rv);
+      LOG(("The match from %s needs to be completed at %s",
+           result.mTableName.get(), gethashUrl.get()));
+      // gethashUrls may be empty in 2 cases: test tables, and on startup where
+      // we may have found a prefix in an existing table before the listmanager
+      // has registered the table. In the second case we should not call
+      // complete.
+      if ((!gethashUrl.IsEmpty() ||
+           StringBeginsWith(result.mTableName, NS_LITERAL_CSTRING("test-"))) &&
+          mDBService->GetCompleter(result.mTableName,
+                                   getter_AddRefs(completer))) {
+        nsAutoCString partialHash;
+        partialHash.Assign(reinterpret_cast<char*>(&result.hash.prefix),
+                           PREFIX_SIZE);
+
+        nsresult rv = completer->Complete(partialHash, gethashUrl, this);
+        if (NS_SUCCEEDED(rv)) {
+          mPendingCompletions++;
+        }
+      } else {
+        // For tables with no hash completer, a complete hash match is
+        // good enough, we'll consider it fresh, even if it hasn't been updated
+        // in 45 minutes.
+        if (result.Complete()) {
+          result.mFresh = true;
+          LOG(("Skipping completion in a table without a valid completer (%s).",
+               result.mTableName.get()));
+        } else {
+          NS_WARNING("Partial match in a table without a valid completer, ignoring partial match.");
+        }
+      }
+    }
+  }
+
+  LOG(("nsUrlClassifierLookupCallback::LookupComplete [%p] "
+       "%u pending completions", this, mPendingCompletions));
+  if (mPendingCompletions == 0) {
+    // All results were complete, we're ready!
+    HandleResults();
+  }
+
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+nsUrlClassifierLookupCallback::CompletionFinished(nsresult status)
+{
+  if (LOG_ENABLED()) {
+    nsAutoCString errorName;
+    mozilla::GetErrorName(status, errorName);
+    LOG(("nsUrlClassifierLookupCallback::CompletionFinished [%p, %s]",
+         this, errorName.get()));
+  }
+
+  mPendingCompletions--;
+  if (mPendingCompletions == 0) {
+    HandleResults();
+  }
+
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+nsUrlClassifierLookupCallback::Completion(const nsACString& completeHash,
+                                          const nsACString& tableName,
+                                          uint32_t chunkId)
+{
+  LOG(("nsUrlClassifierLookupCallback::Completion [%p, %s, %d]",
+       this, PromiseFlatCString(tableName).get(), chunkId));
+  mozilla::safebrowsing::Completion hash;
+  hash.Assign(completeHash);
+
+  // Send this completion to the store for caching.
+  if (!mCacheResults) {
+    mCacheResults = new CacheResultArray();
+    if (!mCacheResults)
+      return NS_ERROR_OUT_OF_MEMORY;
+  }
+
+  CacheResult result;
+  result.entry.addChunk = chunkId;
+  result.entry.complete = hash;
+  result.table = tableName;
+
+  // OK if this fails, we just won't cache the item.
+  mCacheResults->AppendElement(result);
+
+  // Check if this matched any of our results.
+  for (uint32_t i = 0; i < mResults->Length(); i++) {
+    LookupResult& result = mResults->ElementAt(i);
+
+    // Now, see if it verifies a lookup
+    if (!result.mNoise
+        && result.CompleteHash() == hash
+        && result.mTableName.Equals(tableName)) {
+      result.mProtocolConfirmed = true;
+    }
+  }
+
+  return NS_OK;
+}
+
+nsresult
+nsUrlClassifierLookupCallback::HandleResults()
+{
+  if (!mResults) {
+    // No results, this URI is clean.
+    LOG(("nsUrlClassifierLookupCallback::HandleResults [%p, no results]", this));
+    return mCallback->HandleEvent(NS_LITERAL_CSTRING(""));
+  }
+  MOZ_ASSERT(mPendingCompletions == 0, "HandleResults() should never be "
+             "called while there are pending completions");
+
+  LOG(("nsUrlClassifierLookupCallback::HandleResults [%p, %u results]",
+       this, mResults->Length()));
+
+  nsTArray<nsCString> tables;
+  // Build a stringified list of result tables.
+  for (uint32_t i = 0; i < mResults->Length(); i++) {
+    LookupResult& result = mResults->ElementAt(i);
+
+    // Leave out results that weren't confirmed, as their existence on
+    // the list can't be verified.  Also leave out randomly-generated
+    // noise.
+    if (result.mNoise) {
+      LOG(("Skipping result %X from table %s (noise)",
+           result.hash.prefix.ToUint32(), result.mTableName.get()));
+      continue;
+    } else if (!result.Confirmed()) {
+      LOG(("Skipping result %X from table %s (not confirmed)",
+           result.hash.prefix.ToUint32(), result.mTableName.get()));
+      continue;
+    }
+
+    LOG(("Confirmed result %X from table %s",
+         result.hash.prefix.ToUint32(), result.mTableName.get()));
+
+    if (tables.IndexOf(result.mTableName) == nsTArray<nsCString>::NoIndex) {
+      tables.AppendElement(result.mTableName);
+    }
+  }
+
+  // Some parts of this gethash request generated no hits at all.
+  // Prefixes must have been removed from the database since our last update.
+  // Save the prefixes we checked to prevent repeated requests
+  // until the next update.
+  nsAutoPtr<PrefixArray> cacheMisses(new PrefixArray());
+  if (cacheMisses) {
+    for (uint32_t i = 0; i < mResults->Length(); i++) {
+      LookupResult &result = mResults->ElementAt(i);
+      if (!result.Confirmed() && !result.mNoise) {
+        cacheMisses->AppendElement(result.PrefixHash());
+      }
+    }
+    // Hands ownership of the miss array back to the worker thread.
+    mDBService->CacheMisses(cacheMisses.forget());
+  }
+
+  if (mCacheResults) {
+    // This hands ownership of the cache results array back to the worker
+    // thread.
+    mDBService->CacheCompletions(mCacheResults.forget());
+  }
+
+  nsAutoCString tableStr;
+  for (uint32_t i = 0; i < tables.Length(); i++) {
+    if (i != 0)
+      tableStr.Append(',');
+    tableStr.Append(tables[i]);
+  }
+
+  return mCallback->HandleEvent(tableStr);
+}
+
+
+// -------------------------------------------------------------------------
+// Helper class for nsIURIClassifier implementation, translates table names
+// to nsIURIClassifier enums.
+
+class nsUrlClassifierClassifyCallback final : public nsIUrlClassifierCallback
+{
+public:
+  NS_DECL_THREADSAFE_ISUPPORTS
+  NS_DECL_NSIURLCLASSIFIERCALLBACK
+
+  explicit nsUrlClassifierClassifyCallback(nsIURIClassifierCallback *c)
+    : mCallback(c)
+    {}
+
+private:
+  ~nsUrlClassifierClassifyCallback() {}
+
+  nsCOMPtr<nsIURIClassifierCallback> mCallback;
+};
+
+NS_IMPL_ISUPPORTS(nsUrlClassifierClassifyCallback,
+                  nsIUrlClassifierCallback)
+
+NS_IMETHODIMP
+nsUrlClassifierClassifyCallback::HandleEvent(const nsACString& tables)
+{
+  nsresult response = TablesToResponse(tables);
+  mCallback->OnClassifyComplete(response);
+  return NS_OK;
+}
+
+
+// -------------------------------------------------------------------------
+// Proxy class implementation
+
+NS_IMPL_ISUPPORTS(nsUrlClassifierDBService,
+                  nsIUrlClassifierDBService,
+                  nsIURIClassifier,
+                  nsIObserver)
+
+/* static */ nsUrlClassifierDBService*
+nsUrlClassifierDBService::GetInstance(nsresult *result)
+{
+  *result = NS_OK;
+  if (!sUrlClassifierDBService) {
+    sUrlClassifierDBService = new nsUrlClassifierDBService();
+    if (!sUrlClassifierDBService) {
+      *result = NS_ERROR_OUT_OF_MEMORY;
+      return nullptr;
+    }
+
+    NS_ADDREF(sUrlClassifierDBService);   // addref the global
+
+    *result = sUrlClassifierDBService->Init();
+    if (NS_FAILED(*result)) {
+      NS_RELEASE(sUrlClassifierDBService);
+      return nullptr;
+    }
+  } else {
+    // Already exists, just add a ref
+    NS_ADDREF(sUrlClassifierDBService);   // addref the return result
+  }
+  return sUrlClassifierDBService;
+}
+
+
+nsUrlClassifierDBService::nsUrlClassifierDBService()
+ : mCheckMalware(CHECK_MALWARE_DEFAULT)
+ , mCheckPhishing(CHECK_PHISHING_DEFAULT)
+ , mCheckTracking(CHECK_TRACKING_DEFAULT)
+ , mCheckBlockedURIs(CHECK_BLOCKED_DEFAULT)
+ , mInUpdate(false)
+{
+}
+
+nsUrlClassifierDBService::~nsUrlClassifierDBService()
+{
+  sUrlClassifierDBService = nullptr;
+}
+
+nsresult
+nsUrlClassifierDBService::ReadTablesFromPrefs()
+{
+  nsCString allTables;
+  nsCString tables;
+  Preferences::GetCString(PHISH_TABLE_PREF, &allTables);
+
+  Preferences::GetCString(MALWARE_TABLE_PREF, &tables);
+  if (!tables.IsEmpty()) {
+    allTables.Append(',');
+    allTables.Append(tables);
+  }
+
+  Preferences::GetCString(DOWNLOAD_BLOCK_TABLE_PREF, &tables);
+  if (!tables.IsEmpty()) {
+    allTables.Append(',');
+    allTables.Append(tables);
+  }
+
+  Preferences::GetCString(DOWNLOAD_ALLOW_TABLE_PREF, &tables);
+  if (!tables.IsEmpty()) {
+    allTables.Append(',');
+    allTables.Append(tables);
+  }
+
+  Preferences::GetCString(TRACKING_TABLE_PREF, &tables);
+  if (!tables.IsEmpty()) {
+    allTables.Append(',');
+    allTables.Append(tables);
+  }
+
+  Preferences::GetCString(TRACKING_WHITELIST_TABLE_PREF, &tables);
+  if (!tables.IsEmpty()) {
+    allTables.Append(',');
+    allTables.Append(tables);
+  }
+
+  Preferences::GetCString(BLOCKED_TABLE_PREF, &tables);
+  if (!tables.IsEmpty()) {
+    allTables.Append(',');
+    allTables.Append(tables);
+  }
+
+  Classifier::SplitTables(allTables, mGethashTables);
+
+  Preferences::GetCString(DISALLOW_COMPLETION_TABLE_PREF, &tables);
+  Classifier::SplitTables(tables, mDisallowCompletionsTables);
+
+  return NS_OK;
+}
+
+nsresult
+nsUrlClassifierDBService::Init()
+{
+  MOZ_ASSERT(NS_IsMainThread(), "Must initialize DB service on main thread");
+  nsCOMPtr<nsIXULRuntime> appInfo = do_GetService("@mozilla.org/xre/app-info;1");
+  if (appInfo) {
+    bool inSafeMode = false;
+    appInfo->GetInSafeMode(&inSafeMode);
+    if (inSafeMode) {
+      return NS_ERROR_NOT_AVAILABLE;
+    }
+  }
+
+  // Retrieve all the preferences.
+  mCheckMalware = Preferences::GetBool(CHECK_MALWARE_PREF,
+    CHECK_MALWARE_DEFAULT);
+  mCheckPhishing = Preferences::GetBool(CHECK_PHISHING_PREF,
+    CHECK_PHISHING_DEFAULT);
+  mCheckTracking =
+    Preferences::GetBool(CHECK_TRACKING_PREF, CHECK_TRACKING_DEFAULT) ||
+    Preferences::GetBool(CHECK_TRACKING_PB_PREF, CHECK_TRACKING_PB_DEFAULT);
+  mCheckBlockedURIs = Preferences::GetBool(CHECK_BLOCKED_PREF,
+    CHECK_BLOCKED_DEFAULT);
+  uint32_t gethashNoise = Preferences::GetUint(GETHASH_NOISE_PREF,
+    GETHASH_NOISE_DEFAULT);
+  gFreshnessGuarantee = Preferences::GetInt(CONFIRM_AGE_PREF,
+    CONFIRM_AGE_DEFAULT_SEC);
+  ReadTablesFromPrefs();
+
+  nsresult rv;
+
+  {
+    // Force PSM loading on main thread
+    nsCOMPtr<nsICryptoHash> dummy = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv);
+    NS_ENSURE_SUCCESS(rv, rv);
+  }
+
+  {
+    // Force nsIUrlClassifierUtils loading on main thread.
+    nsCOMPtr<nsIUrlClassifierUtils> dummy =
+      do_GetService(NS_URLCLASSIFIERUTILS_CONTRACTID, &rv);
+    NS_ENSURE_SUCCESS(rv, rv);
+  }
+
+  // Directory providers must also be accessed on the main thread.
+  nsCOMPtr<nsIFile> cacheDir;
+  rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_LOCAL_50_DIR,
+                              getter_AddRefs(cacheDir));
+  if (NS_FAILED(rv)) {
+    rv = NS_GetSpecialDirectory(NS_APP_USER_PROFILE_50_DIR,
+                                getter_AddRefs(cacheDir));
+    if (NS_FAILED(rv)) {
+      return rv;
+    }
+  }
+
+  // Start the background thread.
+  rv = NS_NewNamedThread("URL Classifier", &gDbBackgroundThread);
+  if (NS_FAILED(rv))
+    return rv;
+
+  mWorker = new nsUrlClassifierDBServiceWorker();
+  if (!mWorker)
+    return NS_ERROR_OUT_OF_MEMORY;
+
+  rv = mWorker->Init(gethashNoise, cacheDir);
+  if (NS_FAILED(rv)) {
+    mWorker = nullptr;
+    return rv;
+  }
+
+  // Proxy for calling the worker on the background thread
+  mWorkerProxy = new UrlClassifierDBServiceWorkerProxy(mWorker);
+  rv = mWorkerProxy->OpenDb();
+  if (NS_FAILED(rv)) {
+    return rv;
+  }
+
+  // Add an observer for shutdown
+  nsCOMPtr<nsIObserverService> observerService =
+      mozilla::services::GetObserverService();
+  if (!observerService)
+    return NS_ERROR_FAILURE;
+
+  // The application is about to quit
+  observerService->AddObserver(this, "quit-application", false);
+  observerService->AddObserver(this, "profile-before-change", false);
+
+  // XXX: Do we *really* need to be able to change all of these at runtime?
+  // Note: These observers should only be added when everything else above has
+  //       succeeded. Failing to do so can cause long shutdown times in certain
+  //       situations. See Bug 1247798 and Bug 1244803.
+  Preferences::AddStrongObserver(this, CHECK_MALWARE_PREF);
+  Preferences::AddStrongObserver(this, CHECK_PHISHING_PREF);
+  Preferences::AddStrongObserver(this, CHECK_TRACKING_PREF);
+  Preferences::AddStrongObserver(this, CHECK_TRACKING_PB_PREF);
+  Preferences::AddStrongObserver(this, CHECK_BLOCKED_PREF);
+  Preferences::AddStrongObserver(this, GETHASH_NOISE_PREF);
+  Preferences::AddStrongObserver(this, CONFIRM_AGE_PREF);
+  Preferences::AddStrongObserver(this, PHISH_TABLE_PREF);
+  Preferences::AddStrongObserver(this, MALWARE_TABLE_PREF);
+  Preferences::AddStrongObserver(this, TRACKING_TABLE_PREF);
+  Preferences::AddStrongObserver(this, TRACKING_WHITELIST_TABLE_PREF);
+  Preferences::AddStrongObserver(this, BLOCKED_TABLE_PREF);
+  Preferences::AddStrongObserver(this, DOWNLOAD_BLOCK_TABLE_PREF);
+  Preferences::AddStrongObserver(this, DOWNLOAD_ALLOW_TABLE_PREF);
+  Preferences::AddStrongObserver(this, DISALLOW_COMPLETION_TABLE_PREF);
+
+  return NS_OK;
+}
+
+void
+nsUrlClassifierDBService::BuildTables(bool aTrackingProtectionEnabled,
+                                      nsCString &tables)
+{
+  nsAutoCString malware;
+  // LookupURI takes a comma-separated list already.
+  Preferences::GetCString(MALWARE_TABLE_PREF, &malware);
+  if (mCheckMalware && !malware.IsEmpty()) {
+    tables.Append(malware);
+  }
+  nsAutoCString phishing;
+  Preferences::GetCString(PHISH_TABLE_PREF, &phishing);
+  if (mCheckPhishing && !phishing.IsEmpty()) {
+    tables.Append(',');
+    tables.Append(phishing);
+  }
+  if (aTrackingProtectionEnabled) {
+    nsAutoCString tracking, trackingWhitelist;
+    Preferences::GetCString(TRACKING_TABLE_PREF, &tracking);
+    if (!tracking.IsEmpty()) {
+      tables.Append(',');
+      tables.Append(tracking);
+    }
+    Preferences::GetCString(TRACKING_WHITELIST_TABLE_PREF, &trackingWhitelist);
+    if (!trackingWhitelist.IsEmpty()) {
+      tables.Append(',');
+      tables.Append(trackingWhitelist);
+    }
+  }
+  nsAutoCString blocked;
+  Preferences::GetCString(BLOCKED_TABLE_PREF, &blocked);
+  if (mCheckBlockedURIs && !blocked.IsEmpty()) {
+    tables.Append(',');
+    tables.Append(blocked);
+  }
+
+  if (StringBeginsWith(tables, NS_LITERAL_CSTRING(","))) {
+    tables.Cut(0, 1);
+  }
+}
+
+// nsChannelClassifier is the only consumer of this interface.
+NS_IMETHODIMP
+nsUrlClassifierDBService::Classify(nsIPrincipal* aPrincipal,
+                                   bool aTrackingProtectionEnabled,
+                                   nsIURIClassifierCallback* c,
+                                   bool* result)
+{
+  NS_ENSURE_ARG(aPrincipal);
+  NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
+
+  if (!(mCheckMalware || mCheckPhishing || aTrackingProtectionEnabled ||
+        mCheckBlockedURIs)) {
+    *result = false;
+    return NS_OK;
+  }
+
+  RefPtr<nsUrlClassifierClassifyCallback> callback =
+    new nsUrlClassifierClassifyCallback(c);
+  if (!callback) return NS_ERROR_OUT_OF_MEMORY;
+
+  nsAutoCString tables;
+  BuildTables(aTrackingProtectionEnabled, tables);
+
+  nsresult rv = LookupURI(aPrincipal, tables, callback, false, result);
+  if (rv == NS_ERROR_MALFORMED_URI) {
+    *result = false;
+    // The URI had no hostname, don't try to classify it.
+    return NS_OK;
+  }
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+nsUrlClassifierDBService::ClassifyLocalWithTables(nsIURI *aURI,
+                                                  const nsACString & aTables,
+                                                  nsACString & aTableResults)
+{
+  if (gShuttingDownThread) {
+    return NS_ERROR_ABORT;
+  }
+
+  PROFILER_LABEL_FUNC(js::ProfileEntry::Category::OTHER);
+  MOZ_ASSERT(NS_IsMainThread(), "ClassifyLocalWithTables must be on main thread");
+
+  nsCOMPtr<nsIURI> uri = NS_GetInnermostURI(aURI);
+  NS_ENSURE_TRUE(uri, NS_ERROR_FAILURE);
+
+  nsAutoCString key;
+  // Canonicalize the url
+  nsCOMPtr<nsIUrlClassifierUtils> utilsService =
+    do_GetService(NS_URLCLASSIFIERUTILS_CONTRACTID);
+  nsresult rv = utilsService->GetKeyForURI(uri, key);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  nsAutoPtr<LookupResultArray> results(new LookupResultArray());
+  if (!results) {
+    return NS_ERROR_OUT_OF_MEMORY;
+  }
+
+  // In unittests, we may not have been initalized, so don't crash.
+  rv = mWorkerProxy->DoLocalLookup(key, aTables, results);
+  if (NS_SUCCEEDED(rv)) {
+    aTableResults = ProcessLookupResults(results);
+  }
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+nsUrlClassifierDBService::Lookup(nsIPrincipal* aPrincipal,
+                                 const nsACString& tables,
+                                 nsIUrlClassifierCallback* c)
+{
+  NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
+
+  bool dummy;
+  return LookupURI(aPrincipal, tables, c, true, &dummy);
+}
+
+nsresult
+nsUrlClassifierDBService::LookupURI(nsIPrincipal* aPrincipal,
+                                    const nsACString& tables,
+                                    nsIUrlClassifierCallback* c,
+                                    bool forceLookup,
+                                    bool *didLookup)
+{
+  NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
+  NS_ENSURE_ARG(aPrincipal);
+
+  if (nsContentUtils::IsSystemPrincipal(aPrincipal)) {
+    *didLookup = false;
+    return NS_OK;
+  }
+
+  if (gShuttingDownThread) {
+    *didLookup = false;
+    return NS_ERROR_ABORT;
+  }
+
+  nsCOMPtr<nsIURI> uri;
+  nsresult rv = aPrincipal->GetURI(getter_AddRefs(uri));
+  NS_ENSURE_SUCCESS(rv, rv);
+  NS_ENSURE_TRUE(uri, NS_ERROR_FAILURE);
+
+  uri = NS_GetInnermostURI(uri);
+  NS_ENSURE_TRUE(uri, NS_ERROR_FAILURE);
+
+  nsAutoCString key;
+  // Canonicalize the url
+  nsCOMPtr<nsIUrlClassifierUtils> utilsService =
+    do_GetService(NS_URLCLASSIFIERUTILS_CONTRACTID);
+  rv = utilsService->GetKeyForURI(uri, key);
+  if (NS_FAILED(rv))
+    return rv;
+
+  if (forceLookup) {
+    *didLookup = true;
+  } else {
+    bool clean = false;
+
+    if (!clean) {
+      nsCOMPtr<nsIPermissionManager> permissionManager =
+        services::GetPermissionManager();
+
+      if (permissionManager) {
+        uint32_t perm;
+        rv = permissionManager->TestPermissionFromPrincipal(aPrincipal,
+                                                           "safe-browsing", &perm);
+        NS_ENSURE_SUCCESS(rv, rv);
+
+        clean |= (perm == nsIPermissionManager::ALLOW_ACTION);
+      }
+    }
+
+    *didLookup = !clean;
+    if (clean) {
+      return NS_OK;
+    }
+  }
+
+  // Create an nsUrlClassifierLookupCallback object.  This object will
+  // take care of confirming partial hash matches if necessary before
+  // calling the client's callback.
+  nsCOMPtr<nsIUrlClassifierLookupCallback> callback =
+    new nsUrlClassifierLookupCallback(this, c);
+  if (!callback)
+    return NS_ERROR_OUT_OF_MEMORY;
+
+  nsCOMPtr<nsIUrlClassifierLookupCallback> proxyCallback =
+    new UrlClassifierLookupCallbackProxy(callback);
+
+  // Queue this lookup and call the lookup function to flush the queue if
+  // necessary.
+  rv = mWorker->QueueLookup(key, tables, proxyCallback);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  // This seems to just call HandlePendingLookups.
+  nsAutoCString dummy;
+  return mWorkerProxy->Lookup(nullptr, dummy, nullptr);
+}
+
+NS_IMETHODIMP
+nsUrlClassifierDBService::GetTables(nsIUrlClassifierCallback* c)
+{
+  NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
+
+  // The proxy callback uses the current thread.
+  nsCOMPtr<nsIUrlClassifierCallback> proxyCallback =
+    new UrlClassifierCallbackProxy(c);
+
+  return mWorkerProxy->GetTables(proxyCallback);
+}
+
+NS_IMETHODIMP
+nsUrlClassifierDBService::SetHashCompleter(const nsACString &tableName,
+                                           nsIUrlClassifierHashCompleter *completer)
+{
+  if (completer) {
+    mCompleters.Put(tableName, completer);
+  } else {
+    mCompleters.Remove(tableName);
+  }
+  ClearLastResults();
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+nsUrlClassifierDBService::SetLastUpdateTime(const nsACString &tableName,
+                                            uint64_t lastUpdateTime)
+{
+  NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
+
+  return mWorkerProxy->SetLastUpdateTime(tableName, lastUpdateTime);
+}
+
+NS_IMETHODIMP
+nsUrlClassifierDBService::ClearLastResults()
+{
+  NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
+
+  return mWorkerProxy->ClearLastResults();
+}
+
+NS_IMETHODIMP
+nsUrlClassifierDBService::BeginUpdate(nsIUrlClassifierUpdateObserver *observer,
+                                      const nsACString &updateTables)
+{
+  NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
+
+  if (mInUpdate) {
+    LOG(("Already updating, not available"));
+    return NS_ERROR_NOT_AVAILABLE;
+  }
+
+  mInUpdate = true;
+
+  // The proxy observer uses the current thread
+  nsCOMPtr<nsIUrlClassifierUpdateObserver> proxyObserver =
+    new UrlClassifierUpdateObserverProxy(observer);
+
+  return mWorkerProxy->BeginUpdate(proxyObserver, updateTables);
+}
+
+NS_IMETHODIMP
+nsUrlClassifierDBService::BeginStream(const nsACString &table)
+{
+  NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
+
+  return mWorkerProxy->BeginStream(table);
+}
+
+NS_IMETHODIMP
+nsUrlClassifierDBService::UpdateStream(const nsACString& aUpdateChunk)
+{
+  NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
+
+  return mWorkerProxy->UpdateStream(aUpdateChunk);
+}
+
+NS_IMETHODIMP
+nsUrlClassifierDBService::FinishStream()
+{
+  NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
+
+  return mWorkerProxy->FinishStream();
+}
+
+NS_IMETHODIMP
+nsUrlClassifierDBService::FinishUpdate()
+{
+  NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
+
+  mInUpdate = false;
+
+  return mWorkerProxy->FinishUpdate();
+}
+
+
+NS_IMETHODIMP
+nsUrlClassifierDBService::CancelUpdate()
+{
+  NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
+
+  mInUpdate = false;
+
+  return mWorkerProxy->CancelUpdate();
+}
+
+NS_IMETHODIMP
+nsUrlClassifierDBService::ResetDatabase()
+{
+  NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
+
+  return mWorkerProxy->ResetDatabase();
+}
+
+NS_IMETHODIMP
+nsUrlClassifierDBService::ReloadDatabase()
+{
+  NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
+
+  return mWorkerProxy->ReloadDatabase();
+}
+
+nsresult
+nsUrlClassifierDBService::CacheCompletions(CacheResultArray *results)
+{
+  NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
+
+  return mWorkerProxy->CacheCompletions(results);
+}
+
+nsresult
+nsUrlClassifierDBService::CacheMisses(PrefixArray *results)
+{
+  NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
+
+  return mWorkerProxy->CacheMisses(results);
+}
+
+bool
+nsUrlClassifierDBService::GetCompleter(const nsACString &tableName,
+                                       nsIUrlClassifierHashCompleter **completer)
+{
+  // If we have specified a completer, go ahead and query it. This is only
+  // used by tests.
+  if (mCompleters.Get(tableName, completer)) {
+    return true;
+  }
+
+  // If we don't know about this table at all, or are disallowing completions
+  // for it, skip completion checks.
+  if (!mGethashTables.Contains(tableName) ||
+      mDisallowCompletionsTables.Contains(tableName)) {
+    return false;
+  }
+
+  // Otherwise, call gethash to find the hash completions.
+  return NS_SUCCEEDED(CallGetService(NS_URLCLASSIFIERHASHCOMPLETER_CONTRACTID,
+                                     completer));
+}
+
+NS_IMETHODIMP
+nsUrlClassifierDBService::Observe(nsISupports *aSubject, const char *aTopic,
+                                  const char16_t *aData)
+{
+  if (!strcmp(aTopic, NS_PREFBRANCH_PREFCHANGE_TOPIC_ID)) {
+    nsresult rv;
+    nsCOMPtr<nsIPrefBranch> prefs(do_QueryInterface(aSubject, &rv));
+    NS_ENSURE_SUCCESS(rv, rv);
+    Unused << prefs;
+
+    if (NS_LITERAL_STRING(CHECK_MALWARE_PREF).Equals(aData)) {
+      mCheckMalware = Preferences::GetBool(CHECK_MALWARE_PREF,
+        CHECK_MALWARE_DEFAULT);
+    } else if (NS_LITERAL_STRING(CHECK_PHISHING_PREF).Equals(aData)) {
+      mCheckPhishing = Preferences::GetBool(CHECK_PHISHING_PREF,
+        CHECK_PHISHING_DEFAULT);
+    } else if (NS_LITERAL_STRING(CHECK_TRACKING_PREF).Equals(aData) ||
+               NS_LITERAL_STRING(CHECK_TRACKING_PB_PREF).Equals(aData)) {
+      mCheckTracking =
+        Preferences::GetBool(CHECK_TRACKING_PREF, CHECK_TRACKING_DEFAULT) ||
+        Preferences::GetBool(CHECK_TRACKING_PB_PREF, CHECK_TRACKING_PB_DEFAULT);
+    } else if (NS_LITERAL_STRING(CHECK_BLOCKED_PREF).Equals(aData)) {
+      mCheckBlockedURIs = Preferences::GetBool(CHECK_BLOCKED_PREF,
+        CHECK_BLOCKED_DEFAULT);
+    } else if (
+      NS_LITERAL_STRING(PHISH_TABLE_PREF).Equals(aData) ||
+      NS_LITERAL_STRING(MALWARE_TABLE_PREF).Equals(aData) ||
+      NS_LITERAL_STRING(TRACKING_TABLE_PREF).Equals(aData) ||
+      NS_LITERAL_STRING(TRACKING_WHITELIST_TABLE_PREF).Equals(aData) ||
+      NS_LITERAL_STRING(BLOCKED_TABLE_PREF).Equals(aData) ||
+      NS_LITERAL_STRING(DOWNLOAD_BLOCK_TABLE_PREF).Equals(aData) ||
+      NS_LITERAL_STRING(DOWNLOAD_ALLOW_TABLE_PREF).Equals(aData) ||
+      NS_LITERAL_STRING(DISALLOW_COMPLETION_TABLE_PREF).Equals(aData)) {
+      // Just read everything again.
+      ReadTablesFromPrefs();
+    } else if (NS_LITERAL_STRING(CONFIRM_AGE_PREF).Equals(aData)) {
+      gFreshnessGuarantee = Preferences::GetInt(CONFIRM_AGE_PREF,
+        CONFIRM_AGE_DEFAULT_SEC);
+    }
+  } else if (!strcmp(aTopic, "quit-application")) {
+    Shutdown();
+  } else if (!strcmp(aTopic, "profile-before-change")) {
+    // Unit test does not receive "quit-application",
+    // need call shutdown in this case
+    Shutdown();
+    LOG(("joining background thread"));
+    mWorkerProxy = nullptr;
+
+    if (!gDbBackgroundThread) {
+      return NS_OK;
+    }
+
+    nsIThread *backgroundThread = gDbBackgroundThread;
+    gDbBackgroundThread = nullptr;
+    backgroundThread->Shutdown();
+    NS_RELEASE(backgroundThread);
+  } else {
+    return NS_ERROR_UNEXPECTED;
+  }
+
+  return NS_OK;
+}
+
+// Join the background thread if it exists.
+nsresult
+nsUrlClassifierDBService::Shutdown()
+{
+  LOG(("shutting down db service\n"));
+
+  if (!gDbBackgroundThread || gShuttingDownThread) {
+    return NS_OK;
+  }
+
+  gShuttingDownThread = true;
+
+  Telemetry::AutoTimer<Telemetry::URLCLASSIFIER_SHUTDOWN_TIME> timer;
+
+  mCompleters.Clear();
+
+  nsCOMPtr<nsIPrefBranch> prefs = do_GetService(NS_PREFSERVICE_CONTRACTID);
+  if (prefs) {
+    prefs->RemoveObserver(CHECK_MALWARE_PREF, this);
+    prefs->RemoveObserver(CHECK_PHISHING_PREF, this);
+    prefs->RemoveObserver(CHECK_TRACKING_PREF, this);
+    prefs->RemoveObserver(CHECK_TRACKING_PB_PREF, this);
+    prefs->RemoveObserver(CHECK_BLOCKED_PREF, this);
+    prefs->RemoveObserver(PHISH_TABLE_PREF, this);
+    prefs->RemoveObserver(MALWARE_TABLE_PREF, this);
+    prefs->RemoveObserver(TRACKING_TABLE_PREF, this);
+    prefs->RemoveObserver(TRACKING_WHITELIST_TABLE_PREF, this);
+    prefs->RemoveObserver(BLOCKED_TABLE_PREF, this);
+    prefs->RemoveObserver(DOWNLOAD_BLOCK_TABLE_PREF, this);
+    prefs->RemoveObserver(DOWNLOAD_ALLOW_TABLE_PREF, this);
+    prefs->RemoveObserver(DISALLOW_COMPLETION_TABLE_PREF, this);
+    prefs->RemoveObserver(CONFIRM_AGE_PREF, this);
+  }
+
+  DebugOnly<nsresult> rv;
+  // First close the db connection.
+  if (mWorker) {
+    rv = mWorkerProxy->CancelUpdate();
+    NS_ASSERTION(NS_SUCCEEDED(rv), "failed to post cancel update event");
+
+    rv = mWorkerProxy->CloseDb();
+    NS_ASSERTION(NS_SUCCEEDED(rv), "failed to post close db event");
+  }
+  return NS_OK;
+}
+
+nsIThread*
+nsUrlClassifierDBService::BackgroundThread()
+{
+  return gDbBackgroundThread;
+}
+
+// static
+bool
+nsUrlClassifierDBService::ShutdownHasStarted()
+{
+  return gShuttingDownThread;
+}