Add m-esr52 at 52.6.0

1 files changed, 601 insertions, 0 deletions

diff --git a/toolkit/components/url-classifier/content/listmanager.js b/toolkit/components/url-classifier/content/listmanager.js
new file mode 100644
index 000000000..68325bec8
--- /dev/null
+++ b/toolkit/components/url-classifier/content/listmanager.js
@@ -0,0 +1,601 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+var Cu = Components.utils;
+Cu.import("resource://gre/modules/Services.jsm");
+
+// This is the only implementation of nsIUrlListManager.
+// A class that manages lists, namely white and black lists for
+// phishing or malware protection. The ListManager knows how to fetch,
+// update, and store lists.
+//
+// There is a single listmanager for the whole application.
+//
+// TODO more comprehensive update tests, for example add unittest check
+//      that the listmanagers tables are properly written on updates
+
+// Lower and upper limits on the server-provided polling frequency
+const minDelayMs = 5 * 60 * 1000;
+const maxDelayMs = 24 * 60 * 60 * 1000;
+
+// Log only if browser.safebrowsing.debug is true
+this.log = function log(...stuff) {
+  var prefs_ = new G_Preferences();
+  var debug = prefs_.getPref("browser.safebrowsing.debug");
+  if (!debug) {
+    return;
+  }
+
+  var d = new Date();
+  let msg = "listmanager: " + d.toTimeString() + ": " + stuff.join(" ");
+  msg = Services.urlFormatter.trimSensitiveURLs(msg);
+  Services.console.logStringMessage(msg);
+  dump(msg + "\n");
+}
+
+this.QueryAdapter = function QueryAdapter(callback) {
+  this.callback_ = callback;
+};
+
+QueryAdapter.prototype.handleResponse = function(value) {
+  this.callback_.handleEvent(value);
+}
+
+/**
+ * A ListManager keeps track of black and white lists and knows
+ * how to update them.
+ *
+ * @constructor
+ */
+this.PROT_ListManager = function PROT_ListManager() {
+  log("Initializing list manager");
+  this.prefs_ = new G_Preferences();
+  this.updateInterval = this.prefs_.getPref("urlclassifier.updateinterval", 30 * 60) * 1000;
+
+  // A map of tableNames to objects of type
+  // { updateUrl: <updateUrl>, gethashUrl: <gethashUrl> }
+  this.tablesData = {};
+  // A map of updateUrls to maps of tables requiring updates, e.g.
+  // { safebrowsing-update-url: { goog-phish-shavar: true,
+  //                              goog-malware-shavar: true }
+  this.needsUpdate_ = {};
+
+  this.observerServiceObserver_ = new G_ObserverServiceObserver(
+                                          'quit-application',
+                                          BindToObject(this.shutdown_, this),
+                                          true /*only once*/);
+
+  // A map of updateUrls to single-use G_Alarms. An entry exists if and only if
+  // there is at least one table with updates enabled for that url. G_Alarms
+  // are reset when enabling/disabling updates or on update callbacks (update
+  // success, update failure, download error).
+  this.updateCheckers_ = {};
+  this.requestBackoffs_ = {};
+  this.dbService_ = Cc["@mozilla.org/url-classifier/dbservice;1"]
+                   .getService(Ci.nsIUrlClassifierDBService);
+
+
+  this.hashCompleter_ = Cc["@mozilla.org/url-classifier/hashcompleter;1"]
+                        .getService(Ci.nsIUrlClassifierHashCompleter);
+}
+
+/**
+ * xpcom-shutdown callback
+ * Delete all of our data tables which seem to leak otherwise.
+ */
+PROT_ListManager.prototype.shutdown_ = function() {
+  for (var name in this.tablesData) {
+    delete this.tablesData[name];
+  }
+}
+
+/**
+ * Register a new table table
+ * @param tableName - the name of the table
+ * @param updateUrl - the url for updating the table
+ * @param gethashUrl - the url for fetching hash completions
+ * @returns true if the table could be created; false otherwise
+ */
+PROT_ListManager.prototype.registerTable = function(tableName,
+                                                    providerName,
+                                                    updateUrl,
+                                                    gethashUrl) {
+  log("registering " + tableName + " with " + updateUrl);
+  if (!updateUrl) {
+    log("Can't register table " + tableName + " without updateUrl");
+    return false;
+  }
+  this.tablesData[tableName] = {};
+  this.tablesData[tableName].updateUrl = updateUrl;
+  this.tablesData[tableName].gethashUrl = gethashUrl;
+  this.tablesData[tableName].provider = providerName;
+
+  // Keep track of all of our update URLs.
+  if (!this.needsUpdate_[updateUrl]) {
+    this.needsUpdate_[updateUrl] = {};
+
+    // Using the V4 backoff algorithm for both V2 and V4. See bug 1273398.
+    this.requestBackoffs_[updateUrl] = new RequestBackoffV4(
+                                            4 /* num requests */,
+                                   60*60*1000 /* request time, 60 min */);
+  }
+  this.needsUpdate_[updateUrl][tableName] = false;
+
+  return true;
+}
+
+PROT_ListManager.prototype.getGethashUrl = function(tableName) {
+  if (this.tablesData[tableName] && this.tablesData[tableName].gethashUrl) {
+    return this.tablesData[tableName].gethashUrl;
+  }
+  return "";
+}
+
+/**
+ * Enable updates for some tables
+ * @param tables - an array of table names that need updating
+ */
+PROT_ListManager.prototype.enableUpdate = function(tableName) {
+  var table = this.tablesData[tableName];
+  if (table) {
+    log("Enabling table updates for " + tableName);
+    this.needsUpdate_[table.updateUrl][tableName] = true;
+  }
+}
+
+/**
+ * Returns true if any table associated with the updateUrl requires updates.
+ * @param updateUrl - the updateUrl
+ */
+PROT_ListManager.prototype.updatesNeeded_ = function(updateUrl) {
+  let updatesNeeded = false;
+  for (var tableName in this.needsUpdate_[updateUrl]) {
+    if (this.needsUpdate_[updateUrl][tableName]) {
+      updatesNeeded = true;
+    }
+  }
+  return updatesNeeded;
+}
+
+/**
+ * Disables updates for some tables
+ * @param tables - an array of table names that no longer need updating
+ */
+PROT_ListManager.prototype.disableUpdate = function(tableName) {
+  var table = this.tablesData[tableName];
+  if (table) {
+    log("Disabling table updates for " + tableName);
+    this.needsUpdate_[table.updateUrl][tableName] = false;
+    if (!this.updatesNeeded_(table.updateUrl) &&
+        this.updateCheckers_[table.updateUrl]) {
+      this.updateCheckers_[table.updateUrl].cancel();
+      this.updateCheckers_[table.updateUrl] = null;
+    }
+  }
+}
+
+/**
+ * Determine if we have some tables that need updating.
+ */
+PROT_ListManager.prototype.requireTableUpdates = function() {
+  for (var name in this.tablesData) {
+    // Tables that need updating even if other tables don't require it
+    if (this.needsUpdate_[this.tablesData[name].updateUrl][name]) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+/**
+ * Acts as a nsIUrlClassifierCallback for getTables.
+ */
+PROT_ListManager.prototype.kickoffUpdate_ = function (onDiskTableData)
+{
+  this.startingUpdate_ = false;
+  var initialUpdateDelay = 3000;
+  // Add a fuzz of 0-1 minutes for both v2 and v4 according to Bug 1305478.
+  initialUpdateDelay += Math.floor(Math.random() * (1 * 60 * 1000));
+
+  // If the user has never downloaded tables, do the check now.
+  log("needsUpdate: " + JSON.stringify(this.needsUpdate_, undefined, 2));
+  for (var updateUrl in this.needsUpdate_) {
+    // If we haven't already kicked off updates for this updateUrl, set a
+    // non-repeating timer for it. The timer delay will be reset either on
+    // updateSuccess to this.updateInterval, or backed off on downloadError.
+    // Don't set the updateChecker unless at least one table has updates
+    // enabled.
+    if (this.updatesNeeded_(updateUrl) && !this.updateCheckers_[updateUrl]) {
+      let provider = null;
+      Object.keys(this.tablesData).forEach(function(table) {
+        if (this.tablesData[table].updateUrl === updateUrl) {
+          let newProvider = this.tablesData[table].provider;
+          if (provider) {
+            if (newProvider !== provider) {
+              log("Multiple tables for the same updateURL have a different provider?!");
+            }
+          } else {
+            provider = newProvider;
+          }
+        }
+      }, this);
+      log("Initializing update checker for " + updateUrl
+          + " provided by " + provider);
+
+      // Use the initialUpdateDelay + fuzz unless we had previous updates
+      // and the server told us when to try again.
+      let updateDelay = initialUpdateDelay;
+      let targetPref = "browser.safebrowsing.provider." + provider + ".nextupdatetime";
+      let nextUpdate = this.prefs_.getPref(targetPref);
+      if (nextUpdate) {
+        updateDelay = Math.min(maxDelayMs, Math.max(0, nextUpdate - Date.now()));
+        log("Next update at " + nextUpdate);
+      }
+      log("Next update " + updateDelay + "ms from now");
+
+      // Set the last update time to verify if data is still valid.
+      let freshnessPref = "browser.safebrowsing.provider." + provider + ".lastupdatetime";
+      let freshness = this.prefs_.getPref(freshnessPref);
+      if (freshness) {
+        Object.keys(this.tablesData).forEach(function(table) {
+        if (this.tablesData[table].provider === provider) {
+          this.dbService_.setLastUpdateTime(table, freshness);
+        }}, this);
+      }
+
+      this.updateCheckers_[updateUrl] =
+        new G_Alarm(BindToObject(this.checkForUpdates, this, updateUrl),
+                    updateDelay, false /* repeating */);
+    } else {
+      log("No updates needed or already initialized for " + updateUrl);
+    }
+  }
+}
+
+PROT_ListManager.prototype.stopUpdateCheckers = function() {
+  log("Stopping updates");
+  for (var updateUrl in this.updateCheckers_) {
+    if (this.updateCheckers_[updateUrl]) {
+      this.updateCheckers_[updateUrl].cancel();
+      this.updateCheckers_[updateUrl] = null;
+    }
+  }
+}
+
+/**
+ * Determine if we have any tables that require updating.  Different
+ * Wardens may call us with new tables that need to be updated.
+ */
+PROT_ListManager.prototype.maybeToggleUpdateChecking = function() {
+  // We update tables if we have some tables that want updates.  If there
+  // are no tables that want to be updated - we dont need to check anything.
+  if (this.requireTableUpdates()) {
+    log("Starting managing lists");
+
+    // Get the list of existing tables from the DBService before making any
+    // update requests.
+    if (!this.startingUpdate_) {
+      this.startingUpdate_ = true;
+      // check the current state of tables in the database
+      this.dbService_.getTables(BindToObject(this.kickoffUpdate_, this));
+    }
+  } else {
+    log("Stopping managing lists (if currently active)");
+    this.stopUpdateCheckers();                    // Cancel pending updates
+  }
+}
+
+/**
+ * Provides an exception free way to look up the data in a table. We
+ * use this because at certain points our tables might not be loaded,
+ * and querying them could throw.
+ *
+ * @param table String Name of the table that we want to consult
+ * @param key Principal being used to lookup the database
+ * @param callback nsIUrlListManagerCallback (ie., Function) given false or the
+ *        value in the table corresponding to key.  If the table name does not
+ *        exist, we return false, too.
+ */
+PROT_ListManager.prototype.safeLookup = function(key, callback) {
+  try {
+    log("safeLookup: " + key);
+    var cb = new QueryAdapter(callback);
+    this.dbService_.lookup(key,
+                           BindToObject(cb.handleResponse, cb),
+                           true);
+  } catch(e) {
+    log("safeLookup masked failure for key " + key + ": " + e);
+    callback.handleEvent("");
+  }
+}
+
+/**
+ * Updates our internal tables from the update server
+ *
+ * @param updateUrl: request updates for tables associated with that url, or
+ * for all tables if the url is empty.
+ */
+PROT_ListManager.prototype.checkForUpdates = function(updateUrl) {
+  log("checkForUpdates with " + updateUrl);
+  // See if we've triggered the request backoff logic.
+  if (!updateUrl) {
+    return false;
+  }
+  if (!this.requestBackoffs_[updateUrl] ||
+      !this.requestBackoffs_[updateUrl].canMakeRequest()) {
+    log("Can't make update request");
+    return false;
+  }
+  // Grab the current state of the tables from the database
+  this.dbService_.getTables(BindToObject(this.makeUpdateRequest_, this,
+                            updateUrl));
+  return true;
+}
+
+/**
+ * Method that fires the actual HTTP update request.
+ * First we reset any tables that have disappeared.
+ * @param tableData List of table data already in the database, in the form
+ *        tablename;<chunk ranges>\n
+ */
+PROT_ListManager.prototype.makeUpdateRequest_ = function(updateUrl, tableData) {
+  log("this.tablesData: " + JSON.stringify(this.tablesData, undefined, 2));
+  log("existing chunks: " + tableData + "\n");
+  // Disallow blank updateUrls
+  if (!updateUrl) {
+    return;
+  }
+  // An object of the form
+  // { tableList: comma-separated list of tables to request,
+  //   tableNames: map of tables that need updating,
+  //   request: list of tables and existing chunk ranges from tableData
+  // }
+  var streamerMap = { tableList: null,
+                      tableNames: {},
+                      requestPayload: "",
+                      isPostRequest: true };
+
+  let useProtobuf = false;
+  let onceThru = false;
+  for (var tableName in this.tablesData) {
+    // Skip tables not matching this update url
+    if (this.tablesData[tableName].updateUrl != updateUrl) {
+      continue;
+    }
+
+    // Check if |updateURL| is for 'proto'. (only v4 uses protobuf for now.)
+    // We use the table name 'goog-*-proto' and an additional provider "google4"
+    // to describe the v4 settings.
+    let isCurTableProto = tableName.endsWith('-proto');
+    if (!onceThru) {
+      useProtobuf = isCurTableProto;
+      onceThru = true;
+    } else if (useProtobuf !== isCurTableProto) {
+      log('ERROR: Cannot mix "proto" tables with other types ' +
+          'within the same provider.');
+    }
+
+    if (this.needsUpdate_[this.tablesData[tableName].updateUrl][tableName]) {
+      streamerMap.tableNames[tableName] = true;
+    }
+    if (!streamerMap.tableList) {
+      streamerMap.tableList = tableName;
+    } else {
+      streamerMap.tableList += "," + tableName;
+    }
+  }
+
+  if (useProtobuf) {
+    let tableArray = [];
+    Object.keys(streamerMap.tableNames).forEach(aTableName => {
+      if (streamerMap.tableNames[aTableName]) {
+        tableArray.push(aTableName);
+      }
+    });
+
+    // Build the <tablename, stateBase64> mapping.
+    let tableState = {};
+    tableData.split("\n").forEach(line => {
+      let p = line.indexOf(";");
+      if (-1 === p) {
+        return;
+      }
+      let tableName = line.substring(0, p);
+      let metadata = line.substring(p + 1).split(":");
+      let stateBase64 = metadata[0];
+      log(tableName + " ==> " + stateBase64);
+      tableState[tableName] = stateBase64;
+    });
+
+    // The state is a byte stream which server told us from the
+    // last table update. The state would be used to do the partial
+    // update and the empty string means the table has
+    // never been downloaded. See Bug 1287058 for supporting
+    // partial update.
+    let stateArray = [];
+    tableArray.forEach(listName => {
+      stateArray.push(tableState[listName] || "");
+    });
+
+    log("stateArray: " + stateArray);
+
+    let urlUtils = Cc["@mozilla.org/url-classifier/utils;1"]
+                     .getService(Ci.nsIUrlClassifierUtils);
+
+    streamerMap.requestPayload = urlUtils.makeUpdateRequestV4(tableArray,
+                                                              stateArray,
+                                                              tableArray.length);
+    streamerMap.isPostRequest = false;
+  } else {
+    // Build the request. For each table already in the database, include the
+    // chunk data from the database
+    var lines = tableData.split("\n");
+    for (var i = 0; i < lines.length; i++) {
+      var fields = lines[i].split(";");
+      var name = fields[0];
+      if (streamerMap.tableNames[name]) {
+        streamerMap.requestPayload += lines[i] + "\n";
+        delete streamerMap.tableNames[name];
+      }
+    }
+    // For each requested table that didn't have chunk data in the database,
+    // request it fresh
+    for (let tableName in streamerMap.tableNames) {
+      streamerMap.requestPayload += tableName + ";\n";
+    }
+
+    streamerMap.isPostRequest = true;
+  }
+
+  log("update request: " + JSON.stringify(streamerMap, undefined, 2) + "\n");
+
+  // Don't send an empty request.
+  if (streamerMap.requestPayload.length > 0) {
+    this.makeUpdateRequestForEntry_(updateUrl, streamerMap.tableList,
+                                    streamerMap.requestPayload,
+                                    streamerMap.isPostRequest);
+  } else {
+    // We were disabled between kicking off getTables and now.
+    log("Not sending empty request");
+  }
+}
+
+PROT_ListManager.prototype.makeUpdateRequestForEntry_ = function(updateUrl,
+                                                                 tableList,
+                                                                 requestPayload,
+                                                                 isPostRequest) {
+  log("makeUpdateRequestForEntry_: requestPayload " + requestPayload +
+      " update: " + updateUrl + " tablelist: " + tableList + "\n");
+  var streamer = Cc["@mozilla.org/url-classifier/streamupdater;1"]
+                 .getService(Ci.nsIUrlClassifierStreamUpdater);
+
+  this.requestBackoffs_[updateUrl].noteRequest();
+
+  if (!streamer.downloadUpdates(
+        tableList,
+        requestPayload,
+        isPostRequest,
+        updateUrl,
+        BindToObject(this.updateSuccess_, this, tableList, updateUrl),
+        BindToObject(this.updateError_, this, tableList, updateUrl),
+        BindToObject(this.downloadError_, this, tableList, updateUrl))) {
+    // Our alarm gets reset in one of the 3 callbacks.
+    log("pending update, queued request until later");
+  }
+}
+
+/**
+ * Callback function if the update request succeeded.
+ * @param waitForUpdate String The number of seconds that the client should
+ *        wait before requesting again.
+ */
+PROT_ListManager.prototype.updateSuccess_ = function(tableList, updateUrl,
+                                                     waitForUpdateSec) {
+  log("update success for " + tableList + " from " + updateUrl + ": " +
+      waitForUpdateSec + "\n");
+
+  // The time unit below are all milliseconds if not specified.
+
+  var delay = 0;
+  if (waitForUpdateSec) {
+    delay = parseInt(waitForUpdateSec, 10) * 1000;
+  }
+  // As long as the delay is something sane (5 min to 1 day), update
+  // our delay time for requesting updates. We always use a non-repeating
+  // timer since the delay is set differently at every callback.
+  if (delay > maxDelayMs) {
+    log("Ignoring delay from server (too long), waiting " +
+        maxDelayMs + "ms");
+    delay = maxDelayMs;
+  } else if (delay < minDelayMs) {
+    log("Ignoring delay from server (too short), waiting " +
+        this.updateInterval + "ms");
+    delay = this.updateInterval;
+  } else {
+    log("Waiting " + delay + "ms");
+  }
+  this.updateCheckers_[updateUrl] =
+    new G_Alarm(BindToObject(this.checkForUpdates, this, updateUrl),
+                delay, false);
+
+  // Let the backoff object know that we completed successfully.
+  this.requestBackoffs_[updateUrl].noteServerResponse(200);
+
+  // Set last update time for provider
+  // Get the provider for these tables, check for consistency
+  let tables = tableList.split(",");
+  let provider = null;
+  for (let table of tables) {
+    let newProvider = this.tablesData[table].provider;
+    if (provider) {
+      if (newProvider !== provider) {
+        log("Multiple tables for the same updateURL have a different provider?!");
+      }
+    } else {
+      provider = newProvider;
+    }
+  }
+
+  // Store the last update time (needed to know if the table is "fresh")
+  // and the next update time (to know when to update next).
+  let lastUpdatePref = "browser.safebrowsing.provider." + provider + ".lastupdatetime";
+  let now = Date.now();
+  log("Setting last update of " + provider + " to " + now);
+  this.prefs_.setPref(lastUpdatePref, now.toString());
+
+  let nextUpdatePref = "browser.safebrowsing.provider." + provider + ".nextupdatetime";
+  let targetTime = now + delay;
+  log("Setting next update of " + provider + " to " + targetTime
+      + " (" + delay + "ms from now)");
+  this.prefs_.setPref(nextUpdatePref, targetTime.toString());
+}
+
+/**
+ * Callback function if the update request succeeded.
+ * @param result String The error code of the failure
+ */
+PROT_ListManager.prototype.updateError_ = function(table, updateUrl, result) {
+  log("update error for " + table + " from " + updateUrl + ": " + result + "\n");
+  // There was some trouble applying the updates. Don't try again for at least
+  // updateInterval milliseconds.
+  this.updateCheckers_[updateUrl] =
+    new G_Alarm(BindToObject(this.checkForUpdates, this, updateUrl),
+                this.updateInterval, false);
+}
+
+/**
+ * Callback function when the download failed
+ * @param status String http status or an empty string if connection refused.
+ */
+PROT_ListManager.prototype.downloadError_ = function(table, updateUrl, status) {
+  log("download error for " + table + ": " + status + "\n");
+  // If status is empty, then we assume that we got an NS_CONNECTION_REFUSED
+  // error.  In this case, we treat this is a http 500 error.
+  if (!status) {
+    status = 500;
+  }
+  status = parseInt(status, 10);
+  this.requestBackoffs_[updateUrl].noteServerResponse(status);
+  var delay = this.updateInterval;
+  if (this.requestBackoffs_[updateUrl].isErrorStatus(status)) {
+    // Schedule an update for when our backoff is complete
+    delay = this.requestBackoffs_[updateUrl].nextRequestDelay();
+  } else {
+    log("Got non error status for error callback?!");
+  }
+  this.updateCheckers_[updateUrl] =
+    new G_Alarm(BindToObject(this.checkForUpdates, this, updateUrl),
+                delay, false);
+
+}
+
+PROT_ListManager.prototype.QueryInterface = function(iid) {
+  if (iid.equals(Ci.nsISupports) ||
+      iid.equals(Ci.nsIUrlListManager) ||
+      iid.equals(Ci.nsITimerCallback))
+    return this;
+
+  throw Components.results.NS_ERROR_NO_INTERFACE;
+}