Move WebExtensions enabled Add-ons Manager

1 files changed, 1988 insertions, 0 deletions

diff --git a/toolkit/mozapps/webextensions/internal/AddonRepository.jsm b/toolkit/mozapps/webextensions/internal/AddonRepository.jsm
new file mode 100644
index 000000000..7f88d44ad
--- /dev/null
+++ b/toolkit/mozapps/webextensions/internal/AddonRepository.jsm
@@ -0,0 +1,1988 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+const Cc = Components.classes;
+const Ci = Components.interfaces;
+const Cu = Components.utils;
+const Cr = Components.results;
+
+Components.utils.import("resource://gre/modules/Services.jsm");
+Components.utils.import("resource://gre/modules/AddonManager.jsm");
+/* globals AddonManagerPrivate*/
+Components.utils.import("resource://gre/modules/XPCOMUtils.jsm");
+
+XPCOMUtils.defineLazyModuleGetter(this, "NetUtil",
+                                  "resource://gre/modules/NetUtil.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "OS",
+                                  "resource://gre/modules/osfile.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "DeferredSave",
+                                  "resource://gre/modules/DeferredSave.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "AddonRepository_SQLiteMigrator",
+                                  "resource://gre/modules/addons/AddonRepository_SQLiteMigrator.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "Promise",
+                                  "resource://gre/modules/Promise.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "ServiceRequest",
+                                  "resource://gre/modules/ServiceRequest.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "Task",
+                                  "resource://gre/modules/Task.jsm");
+
+
+this.EXPORTED_SYMBOLS = [ "AddonRepository" ];
+
+const PREF_GETADDONS_CACHE_ENABLED       = "extensions.getAddons.cache.enabled";
+const PREF_GETADDONS_CACHE_TYPES         = "extensions.getAddons.cache.types";
+const PREF_GETADDONS_CACHE_ID_ENABLED    = "extensions.%ID%.getAddons.cache.enabled"
+const PREF_GETADDONS_BROWSEADDONS        = "extensions.getAddons.browseAddons";
+const PREF_GETADDONS_BYIDS               = "extensions.getAddons.get.url";
+const PREF_GETADDONS_BYIDS_PERFORMANCE   = "extensions.getAddons.getWithPerformance.url";
+const PREF_GETADDONS_BROWSERECOMMENDED   = "extensions.getAddons.recommended.browseURL";
+const PREF_GETADDONS_GETRECOMMENDED      = "extensions.getAddons.recommended.url";
+const PREF_GETADDONS_BROWSESEARCHRESULTS = "extensions.getAddons.search.browseURL";
+const PREF_GETADDONS_GETSEARCHRESULTS    = "extensions.getAddons.search.url";
+const PREF_GETADDONS_DB_SCHEMA           = "extensions.getAddons.databaseSchema"
+
+const PREF_METADATA_LASTUPDATE           = "extensions.getAddons.cache.lastUpdate";
+const PREF_METADATA_UPDATETHRESHOLD_SEC  = "extensions.getAddons.cache.updateThreshold";
+const DEFAULT_METADATA_UPDATETHRESHOLD_SEC = 172800;  // two days
+
+const XMLURI_PARSE_ERROR  = "http://www.mozilla.org/newlayout/xml/parsererror.xml";
+
+const API_VERSION = "1.5";
+const DEFAULT_CACHE_TYPES = "extension,theme,locale,dictionary";
+
+const KEY_PROFILEDIR        = "ProfD";
+const FILE_DATABASE         = "addons.json";
+const DB_SCHEMA             = 5;
+const DB_MIN_JSON_SCHEMA    = 5;
+const DB_BATCH_TIMEOUT_MS   = 50;
+
+const BLANK_DB = function() {
+  return {
+    addons: new Map(),
+    schema: DB_SCHEMA
+  };
+}
+
+const TOOLKIT_ID     = "toolkit@mozilla.org";
+
+Cu.import("resource://gre/modules/Log.jsm");
+const LOGGER_ID = "addons.repository";
+
+// Create a new logger for use by the Addons Repository
+// (Requires AddonManager.jsm)
+var logger = Log.repository.getLogger(LOGGER_ID);
+
+// A map between XML keys to AddonSearchResult keys for string values
+// that require no extra parsing from XML
+const STRING_KEY_MAP = {
+  name:               "name",
+  version:            "version",
+  homepage:           "homepageURL",
+  support:            "supportURL"
+};
+
+// A map between XML keys to AddonSearchResult keys for string values
+// that require parsing from HTML
+const HTML_KEY_MAP = {
+  summary:            "description",
+  description:        "fullDescription",
+  developer_comments: "developerComments",
+  eula:               "eula"
+};
+
+// A map between XML keys to AddonSearchResult keys for integer values
+// that require no extra parsing from XML
+const INTEGER_KEY_MAP = {
+  total_downloads:  "totalDownloads",
+  weekly_downloads: "weeklyDownloads",
+  daily_users:      "dailyUsers"
+};
+
+function convertHTMLToPlainText(html) {
+  if (!html)
+    return html;
+  var converter = Cc["@mozilla.org/widget/htmlformatconverter;1"].
+                  createInstance(Ci.nsIFormatConverter);
+
+  var input = Cc["@mozilla.org/supports-string;1"].
+              createInstance(Ci.nsISupportsString);
+  input.data = html.replace(/\n/g, "<br>");
+
+  var output = {};
+  converter.convert("text/html", input, input.data.length, "text/unicode",
+                    output, {});
+
+  if (output.value instanceof Ci.nsISupportsString)
+    return output.value.data.replace(/\r\n/g, "\n");
+  return html;
+}
+
+function getAddonsToCache(aIds, aCallback) {
+  try {
+    var types = Services.prefs.getCharPref(PREF_GETADDONS_CACHE_TYPES);
+  }
+  catch (e) { }
+  if (!types)
+    types = DEFAULT_CACHE_TYPES;
+
+  types = types.split(",");
+
+  AddonManager.getAddonsByIDs(aIds, function(aAddons) {
+    let enabledIds = [];
+    for (var i = 0; i < aIds.length; i++) {
+      var preference = PREF_GETADDONS_CACHE_ID_ENABLED.replace("%ID%", aIds[i]);
+      try {
+        if (!Services.prefs.getBoolPref(preference))
+          continue;
+      } catch (e) {
+        // If the preference doesn't exist caching is enabled by default
+      }
+
+      // The add-ons manager may not know about this ID yet if it is a pending
+      // install. In that case we'll just cache it regardless
+      if (aAddons[i] && (types.indexOf(aAddons[i].type) == -1))
+        continue;
+
+      enabledIds.push(aIds[i]);
+    }
+
+    aCallback(enabledIds);
+  });
+}
+
+function AddonSearchResult(aId) {
+  this.id = aId;
+  this.icons = {};
+  this._unsupportedProperties = {};
+}
+
+AddonSearchResult.prototype = {
+  /**
+   * The ID of the add-on
+   */
+  id: null,
+
+  /**
+   * The add-on type (e.g. "extension" or "theme")
+   */
+  type: null,
+
+  /**
+   * The name of the add-on
+   */
+  name: null,
+
+  /**
+   * The version of the add-on
+   */
+  version: null,
+
+  /**
+   * The creator of the add-on
+   */
+  creator: null,
+
+  /**
+   * The developers of the add-on
+   */
+  developers: null,
+
+  /**
+   * A short description of the add-on
+   */
+  description: null,
+
+  /**
+   * The full description of the add-on
+   */
+  fullDescription: null,
+
+  /**
+   * The developer comments for the add-on. This includes any information
+   * that may be helpful to end users that isn't necessarily applicable to
+   * the add-on description (e.g. known major bugs)
+   */
+  developerComments: null,
+
+  /**
+   * The end-user licensing agreement (EULA) of the add-on
+   */
+  eula: null,
+
+  /**
+   * The url of the add-on's icon
+   */
+  get iconURL() {
+    return this.icons && this.icons[32];
+  },
+
+   /**
+   * The URLs of the add-on's icons, as an object with icon size as key
+   */
+  icons: null,
+
+  /**
+   * An array of screenshot urls for the add-on
+   */
+  screenshots: null,
+
+  /**
+   * The homepage for the add-on
+   */
+  homepageURL: null,
+
+  /**
+   * The homepage for the add-on
+   */
+  learnmoreURL: null,
+
+  /**
+   * The support URL for the add-on
+   */
+  supportURL: null,
+
+  /**
+   * The contribution url of the add-on
+   */
+  contributionURL: null,
+
+  /**
+   * The suggested contribution amount
+   */
+  contributionAmount: null,
+
+  /**
+   * The URL to visit in order to purchase the add-on
+   */
+  purchaseURL: null,
+
+  /**
+   * The numerical cost of the add-on in some currency, for sorting purposes
+   * only
+   */
+  purchaseAmount: null,
+
+  /**
+   * The display cost of the add-on, for display purposes only
+   */
+  purchaseDisplayAmount: null,
+
+  /**
+   * The rating of the add-on, 0-5
+   */
+  averageRating: null,
+
+  /**
+   * The number of reviews for this add-on
+   */
+  reviewCount: null,
+
+  /**
+   * The URL to the list of reviews for this add-on
+   */
+  reviewURL: null,
+
+  /**
+   * The total number of times the add-on was downloaded
+   */
+  totalDownloads: null,
+
+  /**
+   * The number of times the add-on was downloaded the current week
+   */
+  weeklyDownloads: null,
+
+  /**
+   * The number of daily users for the add-on
+   */
+  dailyUsers: null,
+
+  /**
+   * AddonInstall object generated from the add-on XPI url
+   */
+  install: null,
+
+  /**
+   * nsIURI storing where this add-on was installed from
+   */
+  sourceURI: null,
+
+  /**
+   * The status of the add-on in the repository (e.g. 4 = "Public")
+   */
+  repositoryStatus: null,
+
+  /**
+   * The size of the add-on's files in bytes. For an add-on that have not yet
+   * been downloaded this may be an estimated value.
+   */
+  size: null,
+
+  /**
+   * The Date that the add-on was most recently updated
+   */
+  updateDate: null,
+
+  /**
+   * True or false depending on whether the add-on is compatible with the
+   * current version of the application
+   */
+  isCompatible: true,
+
+  /**
+   * True or false depending on whether the add-on is compatible with the
+   * current platform
+   */
+  isPlatformCompatible: true,
+
+  /**
+   * Array of AddonCompatibilityOverride objects, that describe overrides for
+   * compatibility with an application versions.
+   **/
+  compatibilityOverrides: null,
+
+  /**
+   * True if the add-on has a secure means of updating
+   */
+  providesUpdatesSecurely: true,
+
+  /**
+   * The current blocklist state of the add-on
+   */
+  blocklistState: Ci.nsIBlocklistService.STATE_NOT_BLOCKED,
+
+  /**
+   * True if this add-on cannot be used in the application based on version
+   * compatibility, dependencies and blocklisting
+   */
+  appDisabled: false,
+
+  /**
+   * True if the user wants this add-on to be disabled
+   */
+  userDisabled: false,
+
+  /**
+   * Indicates what scope the add-on is installed in, per profile, user,
+   * system or application
+   */
+  scope: AddonManager.SCOPE_PROFILE,
+
+  /**
+   * True if the add-on is currently functional
+   */
+  isActive: true,
+
+  /**
+   * A bitfield holding all of the current operations that are waiting to be
+   * performed for this add-on
+   */
+  pendingOperations: AddonManager.PENDING_NONE,
+
+  /**
+   * A bitfield holding all the the operations that can be performed on
+   * this add-on
+   */
+  permissions: 0,
+
+  /**
+   * Tests whether this add-on is known to be compatible with a
+   * particular application and platform version.
+   *
+   * @param  appVersion
+   *         An application version to test against
+   * @param  platformVersion
+   *         A platform version to test against
+   * @return Boolean representing if the add-on is compatible
+   */
+  isCompatibleWith: function(aAppVersion, aPlatformVersion) {
+    return true;
+  },
+
+  /**
+   * Starts an update check for this add-on. This will perform
+   * asynchronously and deliver results to the given listener.
+   *
+   * @param  aListener
+   *         An UpdateListener for the update process
+   * @param  aReason
+   *         A reason code for performing the update
+   * @param  aAppVersion
+   *         An application version to check for updates for
+   * @param  aPlatformVersion
+   *         A platform version to check for updates for
+   */
+  findUpdates: function(aListener, aReason, aAppVersion, aPlatformVersion) {
+    if ("onNoCompatibilityUpdateAvailable" in aListener)
+      aListener.onNoCompatibilityUpdateAvailable(this);
+    if ("onNoUpdateAvailable" in aListener)
+      aListener.onNoUpdateAvailable(this);
+    if ("onUpdateFinished" in aListener)
+      aListener.onUpdateFinished(this);
+  },
+
+  toJSON: function() {
+    let json = {};
+
+    for (let property of Object.keys(this)) {
+      let value = this[property];
+      if (property.startsWith("_") ||
+          typeof(value) === "function")
+        continue;
+
+      try {
+        switch (property) {
+          case "sourceURI":
+            json.sourceURI = value ? value.spec : "";
+            break;
+
+          case "updateDate":
+            json.updateDate = value ? value.getTime() : "";
+            break;
+
+          default:
+            json[property] = value;
+        }
+      } catch (ex) {
+        logger.warn("Error writing property value for " + property);
+      }
+    }
+
+    for (let property of Object.keys(this._unsupportedProperties)) {
+      let value = this._unsupportedProperties[property];
+      if (!property.startsWith("_"))
+        json[property] = value;
+    }
+
+    return json;
+  }
+}
+
+/**
+ * The add-on repository is a source of add-ons that can be installed. It can
+ * be searched in three ways. The first takes a list of IDs and returns a
+ * list of the corresponding add-ons. The second returns a list of add-ons that
+ * come highly recommended. This list should change frequently. The third is to
+ * search for specific search terms entered by the user. Searches are
+ * asynchronous and results should be passed to the provided callback object
+ * when complete. The results passed to the callback should only include add-ons
+ * that are compatible with the current application and are not already
+ * installed.
+ */
+this.AddonRepository = {
+  /**
+   * Whether caching is currently enabled
+   */
+  get cacheEnabled() {
+    let preference = PREF_GETADDONS_CACHE_ENABLED;
+    let enabled = false;
+    try {
+      enabled = Services.prefs.getBoolPref(preference);
+    } catch (e) {
+      logger.warn("cacheEnabled: Couldn't get pref: " + preference);
+    }
+
+    return enabled;
+  },
+
+  // A cache of the add-ons stored in the database
+  _addons: null,
+
+  // Whether a search is currently in progress
+  _searching: false,
+
+  // XHR associated with the current request
+  _request: null,
+
+  /*
+   * Addon search results callback object that contains two functions
+   *
+   * searchSucceeded - Called when a search has suceeded.
+   *
+   * @param  aAddons
+   *         An array of the add-on results. In the case of searching for
+   *         specific terms the ordering of results may be determined by
+   *         the search provider.
+   * @param  aAddonCount
+   *         The length of aAddons
+   * @param  aTotalResults
+   *         The total results actually available in the repository
+   *
+   *
+   * searchFailed - Called when an error occurred when performing a search.
+   */
+  _callback: null,
+
+  // Maximum number of results to return
+  _maxResults: null,
+
+  /**
+   * Shut down AddonRepository
+   * return: promise{integer} resolves with the result of flushing
+   *         the AddonRepository database
+   */
+  shutdown: function() {
+    this.cancelSearch();
+
+    this._addons = null;
+    return AddonDatabase.shutdown(false);
+  },
+
+  metadataAge: function() {
+    let now = Math.round(Date.now() / 1000);
+
+    let lastUpdate = 0;
+    try {
+      lastUpdate = Services.prefs.getIntPref(PREF_METADATA_LASTUPDATE);
+    } catch (e) {}
+
+    // Handle clock jumps
+    if (now < lastUpdate) {
+      return now;
+    }
+    return now - lastUpdate;
+  },
+
+  isMetadataStale: function() {
+    let threshold = DEFAULT_METADATA_UPDATETHRESHOLD_SEC;
+    try {
+      threshold = Services.prefs.getIntPref(PREF_METADATA_UPDATETHRESHOLD_SEC);
+    } catch (e) {}
+    return (this.metadataAge() > threshold);
+  },
+
+  /**
+   * Asynchronously get a cached add-on by id. The add-on (or null if the
+   * add-on is not found) is passed to the specified callback. If caching is
+   * disabled, null is passed to the specified callback.
+   *
+   * @param  aId
+   *         The id of the add-on to get
+   * @param  aCallback
+   *         The callback to pass the result back to
+   */
+  getCachedAddonByID: Task.async(function*(aId, aCallback) {
+    if (!aId || !this.cacheEnabled) {
+      aCallback(null);
+      return;
+    }
+
+    function getAddon(aAddons) {
+      aCallback(aAddons.get(aId) || null);
+    }
+
+    if (this._addons == null) {
+      AddonDatabase.retrieveStoredData().then(aAddons => {
+        this._addons = aAddons;
+        getAddon(aAddons);
+      });
+
+      return;
+    }
+
+    getAddon(this._addons);
+  }),
+
+  /**
+   * Asynchronously repopulate cache so it only contains the add-ons
+   * corresponding to the specified ids. If caching is disabled,
+   * the cache is completely removed.
+   *
+   * @param  aTimeout
+   *         (Optional) timeout in milliseconds to abandon the XHR request
+   *         if we have not received a response from the server.
+   * @return Promise{null}
+   *         Resolves when the metadata ping is complete
+   */
+  repopulateCache: function(aTimeout) {
+    return this._repopulateCacheInternal(false, aTimeout);
+  },
+
+  /*
+   * Clear and delete the AddonRepository database
+   * @return Promise{null} resolves when the database is deleted
+   */
+  _clearCache: function() {
+    this._addons = null;
+    return AddonDatabase.delete().then(() =>
+      new Promise((resolve, reject) =>
+        AddonManagerPrivate.updateAddonRepositoryData(resolve))
+    );
+  },
+
+  _repopulateCacheInternal: Task.async(function*(aSendPerformance, aTimeout) {
+    let allAddons = yield new Promise((resolve, reject) =>
+      AddonManager.getAllAddons(resolve));
+
+    // Filter the hotfix out of our list of add-ons
+    allAddons = allAddons.filter(a => a.id != AddonManager.hotfixID);
+
+    // Completely remove cache if caching is not enabled
+    if (!this.cacheEnabled) {
+      logger.debug("Clearing cache because it is disabled");
+      yield this._clearCache();
+      return;
+    }
+
+    let ids = allAddons.map(a => a.id);
+    logger.debug("Repopulate add-on cache with " + ids.toSource());
+
+    let addonsToCache = yield new Promise((resolve, reject) =>
+      getAddonsToCache(ids, resolve));
+
+    // Completely remove cache if there are no add-ons to cache
+    if (addonsToCache.length == 0) {
+      logger.debug("Clearing cache because 0 add-ons were requested");
+      yield this._clearCache();
+      return;
+    }
+
+    yield new Promise((resolve, reject) =>
+      this._beginGetAddons(addonsToCache, {
+        searchSucceeded: aAddons => {
+          this._addons = new Map();
+          for (let addon of aAddons) {
+            this._addons.set(addon.id, addon);
+          }
+          AddonDatabase.repopulate(aAddons, resolve);
+        },
+        searchFailed: () => {
+          logger.warn("Search failed when repopulating cache");
+          resolve();
+        }
+      }, aSendPerformance, aTimeout));
+
+    // Always call AddonManager updateAddonRepositoryData after we refill the cache
+    yield new Promise((resolve, reject) =>
+      AddonManagerPrivate.updateAddonRepositoryData(resolve));
+  }),
+
+  /**
+   * Asynchronously add add-ons to the cache corresponding to the specified
+   * ids. If caching is disabled, the cache is unchanged and the callback is
+   * immediately called if it is defined.
+   *
+   * @param  aIds
+   *         The array of add-on ids to add to the cache
+   * @param  aCallback
+   *         The optional callback to call once complete
+   */
+  cacheAddons: function(aIds, aCallback) {
+    logger.debug("cacheAddons: enabled " + this.cacheEnabled + " IDs " + aIds.toSource());
+    if (!this.cacheEnabled) {
+      if (aCallback)
+        aCallback();
+      return;
+    }
+
+    getAddonsToCache(aIds, aAddons => {
+      // If there are no add-ons to cache, act as if caching is disabled
+      if (aAddons.length == 0) {
+        if (aCallback)
+          aCallback();
+        return;
+      }
+
+      this.getAddonsByIDs(aAddons, {
+        searchSucceeded: aAddons => {
+          for (let addon of aAddons) {
+            this._addons.set(addon.id, addon);
+          }
+          AddonDatabase.insertAddons(aAddons, aCallback);
+        },
+        searchFailed: () => {
+          logger.warn("Search failed when adding add-ons to cache");
+          if (aCallback)
+            aCallback();
+        }
+      });
+    });
+  },
+
+  /**
+   * The homepage for visiting this repository. If the corresponding preference
+   * is not defined, defaults to about:blank.
+   */
+  get homepageURL() {
+    let url = this._formatURLPref(PREF_GETADDONS_BROWSEADDONS, {});
+    return (url != null) ? url : "about:blank";
+  },
+
+  /**
+   * Returns whether this instance is currently performing a search. New
+   * searches will not be performed while this is the case.
+   */
+  get isSearching() {
+    return this._searching;
+  },
+
+  /**
+   * The url that can be visited to see recommended add-ons in this repository.
+   * If the corresponding preference is not defined, defaults to about:blank.
+   */
+  getRecommendedURL: function() {
+    let url = this._formatURLPref(PREF_GETADDONS_BROWSERECOMMENDED, {});
+    return (url != null) ? url : "about:blank";
+  },
+
+  /**
+   * Retrieves the url that can be visited to see search results for the given
+   * terms. If the corresponding preference is not defined, defaults to
+   * about:blank.
+   *
+   * @param  aSearchTerms
+   *         Search terms used to search the repository
+   */
+  getSearchURL: function(aSearchTerms) {
+    let url = this._formatURLPref(PREF_GETADDONS_BROWSESEARCHRESULTS, {
+      TERMS : encodeURIComponent(aSearchTerms)
+    });
+    return (url != null) ? url : "about:blank";
+  },
+
+  /**
+   * Cancels the search in progress. If there is no search in progress this
+   * does nothing.
+   */
+  cancelSearch: function() {
+    this._searching = false;
+    if (this._request) {
+      this._request.abort();
+      this._request = null;
+    }
+    this._callback = null;
+  },
+
+  /**
+   * Begins a search for add-ons in this repository by ID. Results will be
+   * passed to the given callback.
+   *
+   * @param  aIDs
+   *         The array of ids to search for
+   * @param  aCallback
+   *         The callback to pass results to
+   */
+  getAddonsByIDs: function(aIDs, aCallback) {
+    return this._beginGetAddons(aIDs, aCallback, false);
+  },
+
+  /**
+   * Begins a search of add-ons, potentially sending performance data.
+   *
+   * @param  aIDs
+   *         Array of ids to search for.
+   * @param  aCallback
+   *         Function to pass results to.
+   * @param  aSendPerformance
+   *         Boolean indicating whether to send performance data with the
+   *         request.
+   * @param  aTimeout
+   *         (Optional) timeout in milliseconds to abandon the XHR request
+   *         if we have not received a response from the server.
+   */
+  _beginGetAddons: function(aIDs, aCallback, aSendPerformance, aTimeout) {
+    let ids = aIDs.slice(0);
+
+    let params = {
+      API_VERSION : API_VERSION,
+      IDS : ids.map(encodeURIComponent).join(',')
+    };
+
+    let pref = PREF_GETADDONS_BYIDS;
+
+    if (aSendPerformance) {
+      let type = Services.prefs.getPrefType(PREF_GETADDONS_BYIDS_PERFORMANCE);
+      if (type == Services.prefs.PREF_STRING) {
+        pref = PREF_GETADDONS_BYIDS_PERFORMANCE;
+
+        let startupInfo = Cc["@mozilla.org/toolkit/app-startup;1"].
+                          getService(Ci.nsIAppStartup).
+                          getStartupInfo();
+
+        params.TIME_MAIN = "";
+        params.TIME_FIRST_PAINT = "";
+        params.TIME_SESSION_RESTORED = "";
+        if (startupInfo.process) {
+          if (startupInfo.main) {
+            params.TIME_MAIN = startupInfo.main - startupInfo.process;
+          }
+          if (startupInfo.firstPaint) {
+            params.TIME_FIRST_PAINT = startupInfo.firstPaint -
+                                      startupInfo.process;
+          }
+          if (startupInfo.sessionRestored) {
+            params.TIME_SESSION_RESTORED = startupInfo.sessionRestored -
+                                           startupInfo.process;
+          }
+        }
+      }
+    }
+
+    let url = this._formatURLPref(pref, params);
+
+    let  handleResults = (aElements, aTotalResults, aCompatData) => {
+      // Don't use this._parseAddons() so that, for example,
+      // incompatible add-ons are not filtered out
+      let results = [];
+      for (let i = 0; i < aElements.length && results.length < this._maxResults; i++) {
+        let result = this._parseAddon(aElements[i], null, aCompatData);
+        if (result == null)
+          continue;
+
+        // Ignore add-on if it wasn't actually requested
+        let idIndex = ids.indexOf(result.addon.id);
+        if (idIndex == -1)
+          continue;
+
+        // Ignore add-on if the add-on manager doesn't know about its type:
+        if (!(result.addon.type in AddonManager.addonTypes)) {
+          continue;
+        }
+
+        results.push(result);
+        // Ignore this add-on from now on
+        ids.splice(idIndex, 1);
+      }
+
+      // Include any compatibility overrides for addons not hosted by the
+      // remote repository.
+      for (let id in aCompatData) {
+        let addonCompat = aCompatData[id];
+        if (addonCompat.hosted)
+          continue;
+
+        let addon = new AddonSearchResult(addonCompat.id);
+        // Compatibility overrides can only be for extensions.
+        addon.type = "extension";
+        addon.compatibilityOverrides = addonCompat.compatRanges;
+        let result = {
+          addon: addon,
+          xpiURL: null,
+          xpiHash: null
+        };
+        results.push(result);
+      }
+
+      // aTotalResults irrelevant
+      this._reportSuccess(results, -1);
+    }
+
+    this._beginSearch(url, ids.length, aCallback, handleResults, aTimeout);
+  },
+
+  /**
+   * Performs the daily background update check.
+   *
+   * This API both searches for the add-on IDs specified and sends performance
+   * data. It is meant to be called as part of the daily update ping. It should
+   * not be used for any other purpose. Use repopulateCache instead.
+   *
+   * @return Promise{null} Resolves when the metadata update is complete.
+   */
+  backgroundUpdateCheck: function() {
+    return this._repopulateCacheInternal(true);
+  },
+
+  /**
+   * Begins a search for recommended add-ons in this repository. Results will
+   * be passed to the given callback.
+   *
+   * @param  aMaxResults
+   *         The maximum number of results to return
+   * @param  aCallback
+   *         The callback to pass results to
+   */
+  retrieveRecommendedAddons: function(aMaxResults, aCallback) {
+    let url = this._formatURLPref(PREF_GETADDONS_GETRECOMMENDED, {
+      API_VERSION : API_VERSION,
+
+      // Get twice as many results to account for potential filtering
+      MAX_RESULTS : 2 * aMaxResults
+    });
+
+    let handleResults = (aElements, aTotalResults) => {
+      this._getLocalAddonIds(aLocalAddonIds => {
+        // aTotalResults irrelevant
+        this._parseAddons(aElements, -1, aLocalAddonIds);
+      });
+    }
+
+    this._beginSearch(url, aMaxResults, aCallback, handleResults);
+  },
+
+  /**
+   * Begins a search for add-ons in this repository. Results will be passed to
+   * the given callback.
+   *
+   * @param  aSearchTerms
+   *         The terms to search for
+   * @param  aMaxResults
+   *         The maximum number of results to return
+   * @param  aCallback
+   *         The callback to pass results to
+   */
+  searchAddons: function(aSearchTerms, aMaxResults, aCallback) {
+    let compatMode = "normal";
+    if (!AddonManager.checkCompatibility)
+      compatMode = "ignore";
+    else if (AddonManager.strictCompatibility)
+      compatMode = "strict";
+
+    let substitutions = {
+      API_VERSION : API_VERSION,
+      TERMS : encodeURIComponent(aSearchTerms),
+      // Get twice as many results to account for potential filtering
+      MAX_RESULTS : 2 * aMaxResults,
+      COMPATIBILITY_MODE : compatMode,
+    };
+
+    let url = this._formatURLPref(PREF_GETADDONS_GETSEARCHRESULTS, substitutions);
+
+    let handleResults = (aElements, aTotalResults) => {
+      this._getLocalAddonIds(aLocalAddonIds => {
+        this._parseAddons(aElements, aTotalResults, aLocalAddonIds);
+      });
+    }
+
+    this._beginSearch(url, aMaxResults, aCallback, handleResults);
+  },
+
+  // Posts results to the callback
+  _reportSuccess: function(aResults, aTotalResults) {
+    this._searching = false;
+    this._request = null;
+    // The callback may want to trigger a new search so clear references early
+    let addons = aResults.map(result => result.addon);
+    let callback = this._callback;
+    this._callback = null;
+    callback.searchSucceeded(addons, addons.length, aTotalResults);
+  },
+
+  // Notifies the callback of a failure
+  _reportFailure: function() {
+    this._searching = false;
+    this._request = null;
+    // The callback may want to trigger a new search so clear references early
+    let callback = this._callback;
+    this._callback = null;
+    callback.searchFailed();
+  },
+
+  // Get descendant by unique tag name. Returns null if not unique tag name.
+  _getUniqueDescendant: function(aElement, aTagName) {
+    let elementsList = aElement.getElementsByTagName(aTagName);
+    return (elementsList.length == 1) ? elementsList[0] : null;
+  },
+
+  // Get direct descendant by unique tag name.
+  // Returns null if not unique tag name.
+  _getUniqueDirectDescendant: function(aElement, aTagName) {
+    let elementsList = Array.filter(aElement.children,
+                                    aChild => aChild.tagName == aTagName);
+    return (elementsList.length == 1) ? elementsList[0] : null;
+  },
+
+  // Parse out trimmed text content. Returns null if text content empty.
+  _getTextContent: function(aElement) {
+    let textContent = aElement.textContent.trim();
+    return (textContent.length > 0) ? textContent : null;
+  },
+
+  // Parse out trimmed text content of a descendant with the specified tag name
+  // Returns null if the parsing unsuccessful.
+  _getDescendantTextContent: function(aElement, aTagName) {
+    let descendant = this._getUniqueDescendant(aElement, aTagName);
+    return (descendant != null) ? this._getTextContent(descendant) : null;
+  },
+
+  // Parse out trimmed text content of a direct descendant with the specified
+  // tag name.
+  // Returns null if the parsing unsuccessful.
+  _getDirectDescendantTextContent: function(aElement, aTagName) {
+    let descendant = this._getUniqueDirectDescendant(aElement, aTagName);
+    return (descendant != null) ? this._getTextContent(descendant) : null;
+  },
+
+  /*
+   * Creates an AddonSearchResult by parsing an <addon> element
+   *
+   * @param  aElement
+   *         The <addon> element to parse
+   * @param  aSkip
+   *         Object containing ids and sourceURIs of add-ons to skip.
+   * @param  aCompatData
+   *         Array of parsed addon_compatibility elements to accosiate with the
+   *         resulting AddonSearchResult. Optional.
+   * @return Result object containing the parsed AddonSearchResult, xpiURL and
+   *         xpiHash if the parsing was successful. Otherwise returns null.
+   */
+  _parseAddon: function(aElement, aSkip, aCompatData) {
+    let skipIDs = (aSkip && aSkip.ids) ? aSkip.ids : [];
+    let skipSourceURIs = (aSkip && aSkip.sourceURIs) ? aSkip.sourceURIs : [];
+
+    let guid = this._getDescendantTextContent(aElement, "guid");
+    if (guid == null || skipIDs.indexOf(guid) != -1)
+      return null;
+
+    let addon = new AddonSearchResult(guid);
+    let result = {
+      addon: addon,
+      xpiURL: null,
+      xpiHash: null
+    };
+
+    if (aCompatData && guid in aCompatData)
+      addon.compatibilityOverrides = aCompatData[guid].compatRanges;
+
+    for (let node = aElement.firstChild; node; node = node.nextSibling) {
+      if (!(node instanceof Ci.nsIDOMElement))
+        continue;
+
+      let localName = node.localName;
+
+      // Handle case where the wanted string value is located in text content
+      // but only if the content is not empty
+      if (localName in STRING_KEY_MAP) {
+        addon[STRING_KEY_MAP[localName]] = this._getTextContent(node) || addon[STRING_KEY_MAP[localName]];
+        continue;
+      }
+
+      // Handle case where the wanted string value is html located in text content
+      if (localName in HTML_KEY_MAP) {
+        addon[HTML_KEY_MAP[localName]] = convertHTMLToPlainText(this._getTextContent(node));
+        continue;
+      }
+
+      // Handle case where the wanted integer value is located in text content
+      if (localName in INTEGER_KEY_MAP) {
+        let value = parseInt(this._getTextContent(node));
+        if (value >= 0)
+          addon[INTEGER_KEY_MAP[localName]] = value;
+        continue;
+      }
+
+      // Handle cases that aren't as simple as grabbing the text content
+      switch (localName) {
+        case "type":
+          // Map AMO's type id to corresponding string
+          // https://github.com/mozilla/olympia/blob/master/apps/constants/base.py#L127
+          // These definitions need to be updated whenever AMO adds a new type.
+          let id = parseInt(node.getAttribute("id"));
+          switch (id) {
+            case 1:
+              addon.type = "extension";
+              break;
+            case 2:
+              addon.type = "theme";
+              break;
+            case 3:
+              addon.type = "dictionary";
+              break;
+            case 4:
+              addon.type = "search";
+              break;
+            case 5:
+            case 6:
+              addon.type = "locale";
+              break;
+            case 7:
+              addon.type = "plugin";
+              break;
+            case 8:
+              addon.type = "api";
+              break;
+            case 9:
+              addon.type = "lightweight-theme";
+              break;
+            case 11:
+              addon.type = "webapp";
+              break;
+            default:
+              logger.info("Unknown type id " + id + " found when parsing response for GUID " + guid);
+          }
+          break;
+        case "authors":
+          let authorNodes = node.getElementsByTagName("author");
+          for (let authorNode of authorNodes) {
+            let name = this._getDescendantTextContent(authorNode, "name");
+            let link = this._getDescendantTextContent(authorNode, "link");
+            if (name == null || link == null)
+              continue;
+
+            let author = new AddonManagerPrivate.AddonAuthor(name, link);
+            if (addon.creator == null)
+              addon.creator = author;
+            else {
+              if (addon.developers == null)
+                addon.developers = [];
+
+              addon.developers.push(author);
+            }
+          }
+          break;
+        case "previews":
+          let previewNodes = node.getElementsByTagName("preview");
+          for (let previewNode of previewNodes) {
+            let full = this._getUniqueDescendant(previewNode, "full");
+            if (full == null)
+              continue;
+
+            let fullURL = this._getTextContent(full);
+            let fullWidth = full.getAttribute("width");
+            let fullHeight = full.getAttribute("height");
+
+            let thumbnailURL, thumbnailWidth, thumbnailHeight;
+            let thumbnail = this._getUniqueDescendant(previewNode, "thumbnail");
+            if (thumbnail) {
+              thumbnailURL = this._getTextContent(thumbnail);
+              thumbnailWidth = thumbnail.getAttribute("width");
+              thumbnailHeight = thumbnail.getAttribute("height");
+            }
+            let caption = this._getDescendantTextContent(previewNode, "caption");
+            let screenshot = new AddonManagerPrivate.AddonScreenshot(fullURL, fullWidth, fullHeight,
+                                                                     thumbnailURL, thumbnailWidth,
+                                                                     thumbnailHeight, caption);
+
+            if (addon.screenshots == null)
+              addon.screenshots = [];
+
+            if (previewNode.getAttribute("primary") == 1)
+              addon.screenshots.unshift(screenshot);
+            else
+              addon.screenshots.push(screenshot);
+          }
+          break;
+        case "learnmore":
+          addon.learnmoreURL = this._getTextContent(node);
+          addon.homepageURL = addon.homepageURL || addon.learnmoreURL;
+          break;
+        case "contribution_data":
+          let meetDevelopers = this._getDescendantTextContent(node, "meet_developers");
+          let suggestedAmount = this._getDescendantTextContent(node, "suggested_amount");
+          if (meetDevelopers != null) {
+            addon.contributionURL = meetDevelopers;
+            addon.contributionAmount = suggestedAmount;
+          }
+          break
+        case "payment_data":
+          let link = this._getDescendantTextContent(node, "link");
+          let amountTag = this._getUniqueDescendant(node, "amount");
+          let amount = parseFloat(amountTag.getAttribute("amount"));
+          let displayAmount = this._getTextContent(amountTag);
+          if (link != null && amount != null && displayAmount != null) {
+            addon.purchaseURL = link;
+            addon.purchaseAmount = amount;
+            addon.purchaseDisplayAmount = displayAmount;
+          }
+          break
+        case "rating":
+          let averageRating = parseInt(this._getTextContent(node));
+          if (averageRating >= 0)
+            addon.averageRating = Math.min(5, averageRating);
+          break;
+        case "reviews":
+          let url = this._getTextContent(node);
+          let num = parseInt(node.getAttribute("num"));
+          if (url != null && num >= 0) {
+            addon.reviewURL = url;
+            addon.reviewCount = num;
+          }
+          break;
+        case "status":
+          let repositoryStatus = parseInt(node.getAttribute("id"));
+          if (!isNaN(repositoryStatus))
+            addon.repositoryStatus = repositoryStatus;
+          break;
+        case "all_compatible_os":
+          let nodes = node.getElementsByTagName("os");
+          addon.isPlatformCompatible = Array.some(nodes, function(aNode) {
+            let text = aNode.textContent.toLowerCase().trim();
+            return text == "all" || text == Services.appinfo.OS.toLowerCase();
+          });
+          break;
+        case "install":
+          // No os attribute means the xpi is compatible with any os
+          if (node.hasAttribute("os")) {
+            let os = node.getAttribute("os").trim().toLowerCase();
+            // If the os is not ALL and not the current OS then ignore this xpi
+            if (os != "all" && os != Services.appinfo.OS.toLowerCase())
+              break;
+          }
+
+          let xpiURL = this._getTextContent(node);
+          if (xpiURL == null)
+            break;
+
+          if (skipSourceURIs.indexOf(xpiURL) != -1)
+            return null;
+
+          result.xpiURL = xpiURL;
+          addon.sourceURI = NetUtil.newURI(xpiURL);
+
+          let size = parseInt(node.getAttribute("size"));
+          addon.size = (size >= 0) ? size : null;
+
+          let xpiHash = node.getAttribute("hash");
+          if (xpiHash != null)
+            xpiHash = xpiHash.trim();
+          result.xpiHash = xpiHash ? xpiHash : null;
+          break;
+        case "last_updated":
+          let epoch = parseInt(node.getAttribute("epoch"));
+          if (!isNaN(epoch))
+            addon.updateDate = new Date(1000 * epoch);
+          break;
+        case "icon":
+          addon.icons[node.getAttribute("size")] = this._getTextContent(node);
+          break;
+      }
+    }
+
+    return result;
+  },
+
+  _parseAddons: function(aElements, aTotalResults, aSkip) {
+    let results = [];
+
+    let isSameApplication = aAppNode => this._getTextContent(aAppNode) == Services.appinfo.ID;
+
+    for (let i = 0; i < aElements.length && results.length < this._maxResults; i++) {
+      let element = aElements[i];
+
+      let tags = this._getUniqueDescendant(element, "compatible_applications");
+      if (tags == null)
+        continue;
+
+      let applications = tags.getElementsByTagName("appID");
+      let compatible = Array.some(applications, aAppNode => {
+        if (!isSameApplication(aAppNode))
+          return false;
+
+        let parent = aAppNode.parentNode;
+        let minVersion = this._getDescendantTextContent(parent, "min_version");
+        let maxVersion = this._getDescendantTextContent(parent, "max_version");
+        if (minVersion == null || maxVersion == null)
+          return false;
+
+        let currentVersion = Services.appinfo.version;
+        return (Services.vc.compare(minVersion, currentVersion) <= 0 &&
+                ((!AddonManager.strictCompatibility) ||
+                 Services.vc.compare(currentVersion, maxVersion) <= 0));
+      });
+
+      // Ignore add-ons not compatible with this Application
+      if (!compatible) {
+        if (AddonManager.checkCompatibility)
+          continue;
+
+        if (!Array.some(applications, isSameApplication))
+          continue;
+      }
+
+      // Add-on meets all requirements, so parse out data.
+      // Don't pass in compatiblity override data, because that's only returned
+      // in GUID searches, which don't use _parseAddons().
+      let result = this._parseAddon(element, aSkip);
+      if (result == null)
+        continue;
+
+      // Ignore add-on missing a required attribute
+      let requiredAttributes = ["id", "name", "version", "type", "creator"];
+      if (requiredAttributes.some(aAttribute => !result.addon[aAttribute]))
+        continue;
+
+      // Ignore add-on with a type AddonManager doesn't understand:
+      if (!(result.addon.type in AddonManager.addonTypes))
+        continue;
+
+      // Add only if the add-on is compatible with the platform
+      if (!result.addon.isPlatformCompatible)
+        continue;
+
+      // Add only if there was an xpi compatible with this OS or there was a
+      // way to purchase the add-on
+      if (!result.xpiURL && !result.addon.purchaseURL)
+        continue;
+
+      result.addon.isCompatible = compatible;
+
+      results.push(result);
+      // Ignore this add-on from now on by adding it to the skip array
+      aSkip.ids.push(result.addon.id);
+    }
+
+    // Immediately report success if no AddonInstall instances to create
+    let pendingResults = results.length;
+    if (pendingResults == 0) {
+      this._reportSuccess(results, aTotalResults);
+      return;
+    }
+
+    // Create an AddonInstall for each result
+    for (let result of results) {
+      let addon = result.addon;
+      let callback = aInstall => {
+        addon.install = aInstall;
+        pendingResults--;
+        if (pendingResults == 0)
+          this._reportSuccess(results, aTotalResults);
+      }
+
+      if (result.xpiURL) {
+        AddonManager.getInstallForURL(result.xpiURL, callback,
+                                      "application/x-xpinstall", result.xpiHash,
+                                      addon.name, addon.icons, addon.version);
+      }
+      else {
+        callback(null);
+      }
+    }
+  },
+
+  // Parses addon_compatibility nodes, that describe compatibility overrides.
+  _parseAddonCompatElement: function(aResultObj, aElement) {
+    let guid = this._getDescendantTextContent(aElement, "guid");
+    if (!guid) {
+        logger.debug("Compatibility override is missing guid.");
+      return;
+    }
+
+    let compat = {id: guid};
+    compat.hosted = aElement.getAttribute("hosted") != "false";
+
+    function findMatchingAppRange(aNodes) {
+      let toolkitAppRange = null;
+      for (let node of aNodes) {
+        let appID = this._getDescendantTextContent(node, "appID");
+        if (appID != Services.appinfo.ID && appID != TOOLKIT_ID)
+          continue;
+
+        let minVersion = this._getDescendantTextContent(node, "min_version");
+        let maxVersion = this._getDescendantTextContent(node, "max_version");
+        if (minVersion == null || maxVersion == null)
+          continue;
+
+        let appRange = { appID: appID,
+                         appMinVersion: minVersion,
+                         appMaxVersion: maxVersion };
+
+        // Only use Toolkit app ranges if no ranges match the application ID.
+        if (appID == TOOLKIT_ID)
+          toolkitAppRange = appRange;
+        else
+          return appRange;
+      }
+      return toolkitAppRange;
+    }
+
+    function parseRangeNode(aNode) {
+      let type = aNode.getAttribute("type");
+      // Only "incompatible" (blacklisting) is supported for now.
+      if (type != "incompatible") {
+        logger.debug("Compatibility override of unsupported type found.");
+        return null;
+      }
+
+      let override = new AddonManagerPrivate.AddonCompatibilityOverride(type);
+
+      override.minVersion = this._getDirectDescendantTextContent(aNode, "min_version");
+      override.maxVersion = this._getDirectDescendantTextContent(aNode, "max_version");
+
+      if (!override.minVersion) {
+        logger.debug("Compatibility override is missing min_version.");
+        return null;
+      }
+      if (!override.maxVersion) {
+        logger.debug("Compatibility override is missing max_version.");
+        return null;
+      }
+
+      let appRanges = aNode.querySelectorAll("compatible_applications > application");
+      let appRange = findMatchingAppRange.bind(this)(appRanges);
+      if (!appRange) {
+        logger.debug("Compatibility override is missing a valid application range.");
+        return null;
+      }
+
+      override.appID = appRange.appID;
+      override.appMinVersion = appRange.appMinVersion;
+      override.appMaxVersion = appRange.appMaxVersion;
+
+      return override;
+    }
+
+    let rangeNodes = aElement.querySelectorAll("version_ranges > version_range");
+    compat.compatRanges = Array.map(rangeNodes, parseRangeNode.bind(this))
+                               .filter(aItem => !!aItem);
+    if (compat.compatRanges.length == 0)
+      return;
+
+    aResultObj[compat.id] = compat;
+  },
+
+  // Parses addon_compatibility elements.
+  _parseAddonCompatData: function(aElements) {
+    let compatData = {};
+    Array.forEach(aElements, this._parseAddonCompatElement.bind(this, compatData));
+    return compatData;
+  },
+
+  // Begins a new search if one isn't currently executing
+  _beginSearch: function(aURI, aMaxResults, aCallback, aHandleResults, aTimeout) {
+    if (this._searching || aURI == null || aMaxResults <= 0) {
+      logger.warn("AddonRepository search failed: searching " + this._searching + " aURI " + aURI +
+                  " aMaxResults " + aMaxResults);
+      aCallback.searchFailed();
+      return;
+    }
+
+    this._searching = true;
+    this._callback = aCallback;
+    this._maxResults = aMaxResults;
+
+    logger.debug("Requesting " + aURI);
+
+    this._request = new ServiceRequest();
+    this._request.mozBackgroundRequest = true;
+    this._request.open("GET", aURI, true);
+    this._request.overrideMimeType("text/xml");
+    if (aTimeout) {
+      this._request.timeout = aTimeout;
+    }
+
+    this._request.addEventListener("error", aEvent => this._reportFailure(), false);
+    this._request.addEventListener("timeout", aEvent => this._reportFailure(), false);
+    this._request.addEventListener("load", aEvent => {
+      logger.debug("Got metadata search load event");
+      let request = aEvent.target;
+      let responseXML = request.responseXML;
+
+      if (!responseXML || responseXML.documentElement.namespaceURI == XMLURI_PARSE_ERROR ||
+          (request.status != 200 && request.status != 0)) {
+        this._reportFailure();
+        return;
+      }
+
+      let documentElement = responseXML.documentElement;
+      let elements = documentElement.getElementsByTagName("addon");
+      let totalResults = elements.length;
+      let parsedTotalResults = parseInt(documentElement.getAttribute("total_results"));
+      // Parsed value of total results only makes sense if >= elements.length
+      if (parsedTotalResults >= totalResults)
+        totalResults = parsedTotalResults;
+
+      let compatElements = documentElement.getElementsByTagName("addon_compatibility");
+      let compatData = this._parseAddonCompatData(compatElements);
+
+      aHandleResults(elements, totalResults, compatData);
+    }, false);
+    this._request.send(null);
+  },
+
+  // Gets the id's of local add-ons, and the sourceURI's of local installs,
+  // passing the results to aCallback
+  _getLocalAddonIds: function(aCallback) {
+    let localAddonIds = {ids: null, sourceURIs: null};
+
+    AddonManager.getAllAddons(function(aAddons) {
+      localAddonIds.ids = aAddons.map(a => a.id);
+      if (localAddonIds.sourceURIs)
+        aCallback(localAddonIds);
+    });
+
+    AddonManager.getAllInstalls(function(aInstalls) {
+      localAddonIds.sourceURIs = [];
+      for (let install of aInstalls) {
+        if (install.state != AddonManager.STATE_AVAILABLE)
+          localAddonIds.sourceURIs.push(install.sourceURI.spec);
+      }
+
+      if (localAddonIds.ids)
+        aCallback(localAddonIds);
+    });
+  },
+
+  // Create url from preference, returning null if preference does not exist
+  _formatURLPref: function(aPreference, aSubstitutions) {
+    let url = null;
+    try {
+      url = Services.prefs.getCharPref(aPreference);
+    } catch (e) {
+      logger.warn("_formatURLPref: Couldn't get pref: " + aPreference);
+      return null;
+    }
+
+    url = url.replace(/%([A-Z_]+)%/g, function(aMatch, aKey) {
+      return (aKey in aSubstitutions) ? aSubstitutions[aKey] : aMatch;
+    });
+
+    return Services.urlFormatter.formatURL(url);
+  },
+
+  // Find a AddonCompatibilityOverride that matches a given aAddonVersion and
+  // application/platform version.
+  findMatchingCompatOverride: function(aAddonVersion,
+                                                                     aCompatOverrides,
+                                                                     aAppVersion,
+                                                                     aPlatformVersion) {
+    for (let override of aCompatOverrides) {
+
+      let appVersion = null;
+      if (override.appID == TOOLKIT_ID)
+        appVersion = aPlatformVersion || Services.appinfo.platformVersion;
+      else
+        appVersion = aAppVersion || Services.appinfo.version;
+
+      if (Services.vc.compare(override.minVersion, aAddonVersion) <= 0 &&
+          Services.vc.compare(aAddonVersion, override.maxVersion) <= 0 &&
+          Services.vc.compare(override.appMinVersion, appVersion) <= 0 &&
+          Services.vc.compare(appVersion, override.appMaxVersion) <= 0) {
+        return override;
+      }
+    }
+    return null;
+  },
+
+  flush: function() {
+    return AddonDatabase.flush();
+  }
+};
+
+var AddonDatabase = {
+  connectionPromise: null,
+  // the in-memory database
+  DB: BLANK_DB(),
+
+  /**
+   * A getter to retrieve the path to the DB
+   */
+  get jsonFile() {
+    return OS.Path.join(OS.Constants.Path.profileDir, FILE_DATABASE);
+ },
+
+  /**
+   * Asynchronously opens a new connection to the database file.
+   *
+   * @return {Promise} a promise that resolves to the database.
+   */
+  openConnection: function() {
+    if (!this.connectionPromise) {
+     this.connectionPromise = Task.spawn(function*() {
+       this.DB = BLANK_DB();
+
+       let inputDB, schema;
+
+       try {
+         let data = yield OS.File.read(this.jsonFile, { encoding: "utf-8"})
+         inputDB = JSON.parse(data);
+
+         if (!inputDB.hasOwnProperty("addons") ||
+             !Array.isArray(inputDB.addons)) {
+           throw new Error("No addons array.");
+         }
+
+         if (!inputDB.hasOwnProperty("schema")) {
+           throw new Error("No schema specified.");
+         }
+
+         schema = parseInt(inputDB.schema, 10);
+
+         if (!Number.isInteger(schema) ||
+             schema < DB_MIN_JSON_SCHEMA) {
+           throw new Error("Invalid schema value.");
+         }
+       } catch (e) {
+         if (e instanceof OS.File.Error && e.becauseNoSuchFile) {
+           logger.debug("No " + FILE_DATABASE + " found.");
+         } else {
+           logger.error(`Malformed ${FILE_DATABASE}: ${e} - resetting to empty`);
+         }
+
+         // Create a blank addons.json file
+         this._saveDBToDisk();
+
+         let dbSchema = 0;
+         try {
+           dbSchema = Services.prefs.getIntPref(PREF_GETADDONS_DB_SCHEMA);
+         } catch (e) {}
+
+         if (dbSchema < DB_MIN_JSON_SCHEMA) {
+           let results = yield new Promise((resolve, reject) => {
+             AddonRepository_SQLiteMigrator.migrate(resolve);
+           });
+
+           if (results.length) {
+             yield this._insertAddons(results);
+           }
+
+         }
+
+         Services.prefs.setIntPref(PREF_GETADDONS_DB_SCHEMA, DB_SCHEMA);
+         return this.DB;
+       }
+
+       Services.prefs.setIntPref(PREF_GETADDONS_DB_SCHEMA, DB_SCHEMA);
+
+       // We use _insertAddon manually instead of calling
+       // insertAddons to avoid the write to disk which would
+       // be a waste since this is the data that was just read.
+       for (let addon of inputDB.addons) {
+         this._insertAddon(addon);
+       }
+
+       return this.DB;
+     }.bind(this));
+    }
+
+    return this.connectionPromise;
+  },
+
+  /**
+   * A lazy getter for the database connection.
+   */
+  get connection() {
+    return this.openConnection();
+  },
+
+  /**
+   * Asynchronously shuts down the database connection and releases all
+   * cached objects
+   *
+   * @param  aCallback
+   *         An optional callback to call once complete
+   * @param  aSkipFlush
+   *         An optional boolean to skip flushing data to disk. Useful
+   *         when the database is going to be deleted afterwards.
+   */
+  shutdown: function(aSkipFlush) {
+    if (!this.connectionPromise) {
+      return Promise.resolve();
+    }
+
+    this.connectionPromise = null;
+
+    if (aSkipFlush) {
+      return Promise.resolve();
+    }
+    return this.Writer.flush();
+  },
+
+  /**
+   * Asynchronously deletes the database, shutting down the connection
+   * first if initialized
+   *
+   * @param  aCallback
+   *         An optional callback to call once complete
+   * @return Promise{null} resolves when the database has been deleted
+   */
+  delete: function(aCallback) {
+    this.DB = BLANK_DB();
+
+    this._deleting = this.Writer.flush()
+      .then(null, () => {})
+      // shutdown(true) never rejects
+      .then(() => this.shutdown(true))
+      .then(() => OS.File.remove(this.jsonFile, {}))
+      .then(null, error => logger.error("Unable to delete Addon Repository file " +
+                                 this.jsonFile, error))
+      .then(() => this._deleting = null)
+      .then(aCallback);
+    return this._deleting;
+  },
+
+  toJSON: function() {
+    let json = {
+      schema: this.DB.schema,
+      addons: []
+    }
+
+    for (let [, value] of this.DB.addons)
+      json.addons.push(value);
+
+    return json;
+  },
+
+  /*
+   * This is a deferred task writer that is used
+   * to batch operations done within 50ms of each
+   * other and thus generating only one write to disk
+   */
+  get Writer() {
+    delete this.Writer;
+    this.Writer = new DeferredSave(
+      this.jsonFile,
+      () => { return JSON.stringify(this); },
+      DB_BATCH_TIMEOUT_MS
+    );
+    return this.Writer;
+  },
+
+  /**
+   * Flush any pending I/O on the addons.json file
+   * @return: Promise{null}
+   *          Resolves when the pending I/O (writing out or deleting
+   *          addons.json) completes
+   */
+  flush: function() {
+    if (this._deleting) {
+      return this._deleting;
+    }
+    return this.Writer.flush();
+  },
+
+  /**
+   * Asynchronously retrieve all add-ons from the database
+   * @return: Promise{Map}
+   *          Resolves when the add-ons are retrieved from the database
+   */
+  retrieveStoredData: function() {
+    return this.openConnection().then(db => db.addons);
+  },
+
+  /**
+   * Asynchronously repopulates the database so it only contains the
+   * specified add-ons
+   *
+   * @param  aAddons
+   *         The array of add-ons to repopulate the database with
+   * @param  aCallback
+   *         An optional callback to call once complete
+   */
+  repopulate: function(aAddons, aCallback) {
+    this.DB.addons.clear();
+    this.insertAddons(aAddons, function() {
+      let now = Math.round(Date.now() / 1000);
+      logger.debug("Cache repopulated, setting " + PREF_METADATA_LASTUPDATE + " to " + now);
+      Services.prefs.setIntPref(PREF_METADATA_LASTUPDATE, now);
+      if (aCallback)
+        aCallback();
+    });
+  },
+
+  /**
+   * Asynchronously inserts an array of add-ons into the database
+   *
+   * @param  aAddons
+   *         The array of add-ons to insert
+   * @param  aCallback
+   *         An optional callback to call once complete
+   */
+  insertAddons: Task.async(function*(aAddons, aCallback) {
+    yield this.openConnection();
+    yield this._insertAddons(aAddons, aCallback);
+  }),
+
+  _insertAddons: Task.async(function*(aAddons, aCallback) {
+    for (let addon of aAddons) {
+      this._insertAddon(addon);
+    }
+
+    yield this._saveDBToDisk();
+    aCallback && aCallback();
+  }),
+
+  /**
+   * Inserts an individual add-on into the database. If the add-on already
+   * exists in the database (by id), then the specified add-on will not be
+   * inserted.
+   *
+   * @param  aAddon
+   *         The add-on to insert into the database
+   * @param  aCallback
+   *         The callback to call once complete
+   */
+  _insertAddon: function(aAddon) {
+    let newAddon = this._parseAddon(aAddon);
+    if (!newAddon ||
+        !newAddon.id ||
+        this.DB.addons.has(newAddon.id))
+      return;
+
+    this.DB.addons.set(newAddon.id, newAddon);
+  },
+
+  /*
+   * Creates an AddonSearchResult by parsing an object structure
+   * retrieved from the DB JSON representation.
+   *
+   * @param  aObj
+   *         The object to parse
+   * @return Returns an AddonSearchResult object.
+   */
+  _parseAddon: function(aObj) {
+    if (aObj instanceof AddonSearchResult)
+      return aObj;
+
+    let id = aObj.id;
+    if (!aObj.id)
+      return null;
+
+    let addon = new AddonSearchResult(id);
+
+    for (let expectedProperty of Object.keys(AddonSearchResult.prototype)) {
+      if (!(expectedProperty in aObj) ||
+          typeof(aObj[expectedProperty]) === "function")
+        continue;
+
+      let value = aObj[expectedProperty];
+
+      try {
+        switch (expectedProperty) {
+          case "sourceURI":
+            addon.sourceURI = value ? NetUtil.newURI(value) :  null;
+            break;
+
+          case "creator":
+            addon.creator = value
+                            ? this._makeDeveloper(value)
+                            : null;
+            break;
+
+          case "updateDate":
+            addon.updateDate = value ? new Date(value) : null;
+            break;
+
+          case "developers":
+            if (!addon.developers) addon.developers = [];
+            for (let developer of value) {
+              addon.developers.push(this._makeDeveloper(developer));
+            }
+            break;
+
+          case "screenshots":
+            if (!addon.screenshots) addon.screenshots = [];
+            for (let screenshot of value) {
+              addon.screenshots.push(this._makeScreenshot(screenshot));
+            }
+            break;
+
+          case "compatibilityOverrides":
+            if (!addon.compatibilityOverrides) addon.compatibilityOverrides = [];
+            for (let override of value) {
+              addon.compatibilityOverrides.push(
+                this._makeCompatOverride(override)
+              );
+            }
+            break;
+
+          case "icons":
+            if (!addon.icons) addon.icons = {};
+            for (let size of Object.keys(aObj.icons)) {
+              addon.icons[size] = aObj.icons[size];
+            }
+            break;
+
+          case "iconURL":
+            break;
+
+          default:
+            addon[expectedProperty] = value;
+        }
+      } catch (ex) {
+        logger.warn("Error in parsing property value for " + expectedProperty + " | " + ex);
+      }
+
+      // delete property from obj to indicate we've already
+      // handled it. The remaining public properties will
+      // be stored separately and just passed through to
+      // be written back to the DB.
+      delete aObj[expectedProperty];
+    }
+
+    // Copy remaining properties to a separate object
+    // to prevent accidental access on downgraded versions.
+    // The properties will be merged in the same object
+    // prior to being written back through toJSON.
+    for (let remainingProperty of Object.keys(aObj)) {
+      switch (typeof(aObj[remainingProperty])) {
+        case "boolean":
+        case "number":
+        case "string":
+        case "object":
+          // these types are accepted
+          break;
+        default:
+          continue;
+      }
+
+      if (!remainingProperty.startsWith("_"))
+        addon._unsupportedProperties[remainingProperty] =
+          aObj[remainingProperty];
+    }
+
+    return addon;
+  },
+
+  /**
+   * Write the in-memory DB to disk, after waiting for
+   * the DB_BATCH_TIMEOUT_MS timeout.
+   *
+   * @return Promise A promise that resolves after the
+   *                 write to disk has completed.
+   */
+  _saveDBToDisk: function() {
+    return this.Writer.saveChanges().then(
+      null,
+      e => logger.error("SaveDBToDisk failed", e));
+  },
+
+  /**
+   * Make a developer object from a vanilla
+   * JS object from the JSON database
+   *
+   * @param  aObj
+   *         The JS object to use
+   * @return The created developer
+   */
+  _makeDeveloper: function(aObj) {
+    let name = aObj.name;
+    let url = aObj.url;
+    return new AddonManagerPrivate.AddonAuthor(name, url);
+  },
+
+  /**
+   * Make a screenshot object from a vanilla
+   * JS object from the JSON database
+   *
+   * @param  aObj
+   *         The JS object to use
+   * @return The created screenshot
+   */
+  _makeScreenshot: function(aObj) {
+    let url = aObj.url;
+    let width = aObj.width;
+    let height = aObj.height;
+    let thumbnailURL = aObj.thumbnailURL;
+    let thumbnailWidth = aObj.thumbnailWidth;
+    let thumbnailHeight = aObj.thumbnailHeight;
+    let caption = aObj.caption;
+    return new AddonManagerPrivate.AddonScreenshot(url, width, height, thumbnailURL,
+                                                   thumbnailWidth, thumbnailHeight, caption);
+  },
+
+  /**
+   * Make a CompatibilityOverride from a vanilla
+   * JS object from the JSON database
+   *
+   * @param  aObj
+   *         The JS object to use
+   * @return The created CompatibilityOverride
+   */
+  _makeCompatOverride: function(aObj) {
+    let type = aObj.type;
+    let minVersion = aObj.minVersion;
+    let maxVersion = aObj.maxVersion;
+    let appID = aObj.appID;
+    let appMinVersion = aObj.appMinVersion;
+    let appMaxVersion = aObj.appMaxVersion;
+    return new AddonManagerPrivate.AddonCompatibilityOverride(type,
+                                                              minVersion,
+                                                              maxVersion,
+                                                              appID,
+                                                              appMinVersion,
+                                                              appMaxVersion);
+  },
+};