Bug 1276339: Storage inspector doesn't work on chrome:// pages and web extensions

Issue #31

2 files changed, 221 insertions, 91 deletions

diff --git a/devtools/server/actors/storage.js b/devtools/server/actors/storage.js
index 894e282f0..22f4eaabe 100644
--- a/devtools/server/actors/storage.js
+++ b/devtools/server/actors/storage.js
@@ -2,6 +2,8 @@
  * License, v. 2.0. If a copy of the MPL was not distributed with this
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
+/* globals StopIteration */
+
 "use strict";
 
 const {Cc, Ci, Cu, CC} = require("chrome");
@@ -93,7 +95,7 @@ var StorageActors = {};
  *   - observe : Method which gets triggered on the notificaiton of the watched
  *               topic.
  *   - getNamesForHost : Given a host, get list of all known store names.
- *   - getValuesForHost : Given a host (and optianally a name) get all known
+ *   - getValuesForHost : Given a host (and optionally a name) get all known
  *                        store objects.
  *   - toStoreObject : Given a store object, convert it to the required format
  *                     so that it can be transferred over wire.
@@ -141,6 +143,9 @@ StorageActors.defaults = function (typeName, observationTopic) {
      * Converts the window.location object into host.
      */
     getHostName(location) {
+      if (location.protocol === "chrome:") {
+        return location.href;
+      }
       return location.hostname || location.href;
     },
 
@@ -744,6 +749,7 @@ var cookieHelpers = {
 
     let enumerator =
       Services.cookies.getCookiesFromHost(origHost, data.originAttributes || {});
+
     while (enumerator.hasMoreElements()) {
       let nsiCookie = enumerator.getNext().QueryInterface(Ci.nsICookie2);
       if (nsiCookie.name === origName &&
@@ -1041,6 +1047,9 @@ function getObjectForLocalOrSessionStorage(type) {
       if (!location.host) {
         return location.href;
       }
+      if (location.protocol === "chrome:") {
+        return location.href;
+      }
       return location.protocol + "//" + location.host;
     },
 
@@ -1251,6 +1260,9 @@ StorageActors.createActor({
     if (!location.host) {
       return location.href;
     }
+    if (location.protocol === "chrome:") {
+      return location.href;
+    }
     return location.protocol + "//" + location.host;
   },
 
@@ -1423,12 +1435,15 @@ ObjectStoreMetadata.prototype = {
  *        The host associated with this indexed db.
  * @param {IDBDatabase} db
  *        The particular indexed db.
+ * @param {String} storage
+ *        Storage type, either "temporary", "default" or "persistent".
  */
-function DatabaseMetadata(origin, db) {
+function DatabaseMetadata(origin, db, storage) {
   this._origin = origin;
   this._name = db.name;
   this._version = db.version;
   this._objectStores = [];
+  this.storage = storage;
 
   if (db.objectStoreNames.length) {
     let transaction = db.transaction(db.objectStoreNames, "readonly");
@@ -1448,7 +1463,7 @@ DatabaseMetadata.prototype = {
 
   toObject() {
     return {
-      name: this._name,
+      name: `${this._name} (${this.storage})`,
       origin: this._origin,
       version: this._version,
       objectStores: this._objectStores.size
@@ -1524,6 +1539,9 @@ StorageActors.createActor({
     if (!location.host) {
       return location.href;
     }
+    if (location.protocol === "chrome:") {
+      return location.href;
+    }
     return location.protocol + "//" + location.host;
   },
 
@@ -1616,15 +1634,17 @@ StorageActors.createActor({
   populateStoresForHost: Task.async(function* (host) {
     let storeMap = new Map();
     let {names} = yield this.getDBNamesForHost(host);
+
     let win = this.storageActor.getWindowFromHost(host);
     if (win) {
       let principal = win.document.nodePrincipal;
 
-      for (let name of names) {
-        let metadata = yield this.getDBMetaData(host, principal, name);
+      for (let {name, storage} of names) {
+        let metadata = yield this.getDBMetaData(host, principal, name, storage);
 
         metadata = indexedDBHelpers.patchMetadataMapsAndProtos(metadata);
-        storeMap.set(name, metadata);
+
+        storeMap.set(`${name} (${storage})`, metadata);
       }
     }
 
@@ -1657,10 +1677,22 @@ StorageActors.createActor({
         objectStores: item.objectStores
       };
     }
+
+    let value = JSON.stringify(item.value);
+
+    // FIXME: Bug 1318029 - Due to a bug that is thrown whenever a
+    // LongStringActor string reaches DebuggerServer.LONG_STRING_LENGTH we need
+    // to trim the value. When the bug is fixed we should stop trimming the
+    // string here.
+    let maxLength = DebuggerServer.LONG_STRING_LENGTH - 1;
+    if (value.length > maxLength) {
+      value = value.substr(0, maxLength);
+    }
+
     // Indexed db entry
     return {
       name: item.name,
-      value: new LongStringActor(this.conn, JSON.stringify(item.value))
+      value: new LongStringActor(this.conn, value)
     };
   },
 
@@ -1696,16 +1728,18 @@ StorageActors.createActor({
   maybeSetupChildProcess() {
     if (!DebuggerServer.isInChildProcess) {
       this.backToChild = (func, rv) => rv;
+      this.clearDBStore = indexedDBHelpers.clearDBStore;
+      this.gatherFilesOrFolders = indexedDBHelpers.gatherFilesOrFolders;
       this.getDBMetaData = indexedDBHelpers.getDBMetaData;
-      this.openWithPrincipal = indexedDBHelpers.openWithPrincipal;
       this.getDBNamesForHost = indexedDBHelpers.getDBNamesForHost;
-      this.getSanitizedHost = indexedDBHelpers.getSanitizedHost;
       this.getNameFromDatabaseFile = indexedDBHelpers.getNameFromDatabaseFile;
-      this.getValuesForHost = indexedDBHelpers.getValuesForHost;
       this.getObjectStoreData = indexedDBHelpers.getObjectStoreData;
+      this.getSanitizedHost = indexedDBHelpers.getSanitizedHost;
+      this.getValuesForHost = indexedDBHelpers.getValuesForHost;
+      this.openWithPrincipal = indexedDBHelpers.openWithPrincipal;
       this.removeDB = indexedDBHelpers.removeDB;
       this.removeDBRecord = indexedDBHelpers.removeDBRecord;
-      this.clearDBStore = indexedDBHelpers.clearDBStore;
+      this.splitNameAndStorage = indexedDBHelpers.splitNameAndStorage;
       return;
     }
 
@@ -1718,6 +1752,7 @@ StorageActors.createActor({
     });
 
     this.getDBMetaData = callParentProcessAsync.bind(null, "getDBMetaData");
+    this.splitNameAndStorage = callParentProcessAsync.bind(null, "splitNameAndStorage");
     this.getDBNamesForHost = callParentProcessAsync.bind(null, "getDBNamesForHost");
     this.getValuesForHost = callParentProcessAsync.bind(null, "getValuesForHost");
     this.removeDB = callParentProcessAsync.bind(null, "removeDB");
@@ -1813,14 +1848,14 @@ var indexedDBHelpers = {
    * `name` for the given `host` with its `principal`. The stored metadata
    * information is of `DatabaseMetadata` type.
    */
-  getDBMetaData: Task.async(function* (host, principal, name) {
-    let request = this.openWithPrincipal(principal, name);
+  getDBMetaData: Task.async(function* (host, principal, name, storage) {
+    let request = this.openWithPrincipal(principal, name, storage);
     let success = promise.defer();
 
     request.onsuccess = event => {
       let db = event.target.result;
 
-      let dbData = new DatabaseMetadata(host, db);
+      let dbData = new DatabaseMetadata(host, db, storage);
       db.close();
 
       success.resolve(this.backToChild("getDBMetaData", dbData));
@@ -1833,21 +1868,37 @@ var indexedDBHelpers = {
     return success.promise;
   }),
 
+  splitNameAndStorage: function (name) {
+    let lastOpenBracketIndex = name.lastIndexOf("(");
+    let lastCloseBracketIndex = name.lastIndexOf(")");
+    let delta = lastCloseBracketIndex - lastOpenBracketIndex - 1;
+
+    let storage = name.substr(lastOpenBracketIndex + 1, delta);
+
+    name = name.substr(0, lastOpenBracketIndex - 1);
+
+    return { storage, name };
+  },
+
   /**
    * Opens an indexed db connection for the given `principal` and
    * database `name`.
    */
-  openWithPrincipal(principal, name) {
-    return indexedDBForStorage.openForPrincipal(principal, name);
+  openWithPrincipal: function (principal, name, storage) {
+    return indexedDBForStorage.openForPrincipal(principal, name,
+                                                { storage: storage });
   },
 
-  removeDB: Task.async(function* (host, principal, name) {
+  removeDB: Task.async(function* (host, principal, dbName) {
     let result = new promise(resolve => {
-      let request = indexedDBForStorage.deleteForPrincipal(principal, name);
+      let {name, storage} = this.splitNameAndStorage(dbName);
+      let request =
+        indexedDBForStorage.deleteForPrincipal(principal, name,
+                                               { storage: storage });
 
       request.onsuccess = () => {
         resolve({});
-        this.onItemUpdated("deleted", host, [name]);
+        this.onItemUpdated("deleted", host, [dbName]);
       };
 
       request.onblocked = () => {
@@ -1873,10 +1924,11 @@ var indexedDBHelpers = {
 
   removeDBRecord: Task.async(function* (host, principal, dbName, storeName, id) {
     let db;
+    let {name, storage} = this.splitNameAndStorage(dbName);
 
     try {
       db = yield new promise((resolve, reject) => {
-        let request = this.openWithPrincipal(principal, dbName);
+        let request = this.openWithPrincipal(principal, name, storage);
         request.onsuccess = ev => resolve(ev.target.result);
         request.onerror = ev => reject(ev.target.error);
       });
@@ -1905,10 +1957,11 @@ var indexedDBHelpers = {
 
   clearDBStore: Task.async(function* (host, principal, dbName, storeName) {
     let db;
+    let {name, storage} = this.splitNameAndStorage(dbName);
 
     try {
       db = yield new promise((resolve, reject) => {
-        let request = this.openWithPrincipal(principal, dbName);
+        let request = this.openWithPrincipal(principal, name, storage);
         request.onsuccess = ev => resolve(ev.target.result);
         request.onerror = ev => reject(ev.target.error);
       });
@@ -1940,46 +1993,106 @@ var indexedDBHelpers = {
    */
   getDBNamesForHost: Task.async(function* (host) {
     let sanitizedHost = this.getSanitizedHost(host);
-    let directory = OS.Path.join(OS.Constants.Path.profileDir, "storage",
-                                 "default", sanitizedHost, "idb");
+    let profileDir = OS.Constants.Path.profileDir;
+    let files = [];
+    let names = [];
+    let storagePath = OS.Path.join(profileDir, "storage");
+
+    // We expect sqlite DB paths to look something like this:
+    // - PathToProfileDir/storage/default/http+++www.example.com/
+    //   idb/1556056096MeysDaabta.sqlite
+    // - PathToProfileDir/storage/permanent/http+++www.example.com/
+    //   idb/1556056096MeysDaabta.sqlite
+    // - PathToProfileDir/storage/temporary/http+++www.example.com/
+    //   idb/1556056096MeysDaabta.sqlite
+    //
+    // The subdirectory inside the storage folder is determined by the storage
+    // type:
+    // - default:   { storage: "default" } or not specified.
+    // - permanent: { storage: "persistent" }.
+    // - temporary: { storage: "temporary" }.
+    let sqliteFiles = yield this.gatherFilesOrFolders(storagePath, path => {
+      if (path.endsWith(".sqlite")) {
+        let { components } = OS.Path.split(path);
+        let isIDB = components[components.length - 2] === "idb";
+
+        return isIDB;
+      }
+      return false;
+    });
 
-    let exists = yield OS.File.exists(directory);
-    if (!exists && host.startsWith("about:")) {
-      // try for moz-safe-about directory
-      sanitizedHost = this.getSanitizedHost("moz-safe-" + host);
-      directory = OS.Path.join(OS.Constants.Path.profileDir, "storage",
-                               "permanent", sanitizedHost, "idb");
-      exists = yield OS.File.exists(directory);
-    }
-    if (!exists) {
-      return this.backToChild("getDBNamesForHost", {names: []});
+    for (let file of sqliteFiles) {
+      let splitPath = OS.Path.split(file).components;
+      let idbIndex = splitPath.indexOf("idb");
+      let name = splitPath[idbIndex - 1];
+      let storage = splitPath[idbIndex - 2];
+      let relative = file.substr(profileDir.length + 1);
+
+      if (name.startsWith(sanitizedHost)) {
+        files.push({
+          file: relative,
+          storage: storage === "permanent" ? "persistent" : storage
+        });
+      }
     }
 
-    let names = [];
-    let dirIterator = new OS.File.DirectoryIterator(directory);
-    try {
-      yield dirIterator.forEach(file => {
-        // Skip directories.
-        if (file.isDir) {
-          return null;
+    if (files.length > 0) {
+      for (let {file, storage} of files) {
+        let name = yield this.getNameFromDatabaseFile(file);
+        if (name) {
+          names.push({
+            name,
+            storage
+          });
         }
+      }
+    }
+    return this.backToChild("getDBNamesForHost", {names});
+  }),
 
-        // Skip any non-sqlite files.
-        if (!file.name.endsWith(".sqlite")) {
-          return null;
-        }
+  /**
+   * Gather together all of the files in path and pass each path through a
+   * validation function.
+   *
+   * @param {String}
+   *        Path in which to begin searching.
+   * @param {Function}
+   *        Validation function, which checks each file path. If this function
+   *        Returns true the file path is kept.
+   *
+   * @returns {Array}
+   *          An array of file paths.
+   */
+  gatherFilesOrFolders: Task.async(function* (path, validationFunc) {
+    let files = [];
+    let iterator;
+    let paths = [path];
+
+    while (paths.length > 0) {
+      try {
+        iterator = new OS.File.DirectoryIterator(paths.pop());
 
-        return this.getNameFromDatabaseFile(file.path).then(name => {
-          if (name) {
-            names.push(name);
+        for (let child in iterator) {
+          child = yield child;
+
+          path = child.path;
+
+          if (child.isDir) {
+            paths.push(path);
+          } else if (validationFunc(path)) {
+            files.push(path);
           }
-          return null;
-        });
-      });
-    } finally {
-      dirIterator.close();
+        }
+      } catch (ex) {
+        // Ignore StopIteration to prevent exiting the loop.
+        if (ex != StopIteration) {
+          throw ex;
+        }
+      }
     }
-    return this.backToChild("getDBNamesForHost", {names: names});
+    iterator.close();
+
+    return files;
   }),
 
   /**
@@ -1987,6 +2100,9 @@ var indexedDBHelpers = {
    * name.
    */
   getSanitizedHost(host) {
+    if (host.startsWith("about:")) {
+      host = "moz-safe-" + host;
+    }
     return host.replace(ILLEGAL_CHAR_REGEX, "+");
   },
 
@@ -2000,7 +2116,7 @@ var indexedDBHelpers = {
 
     // Content pages might be having an open transaction for the same indexed db
     // which this sqlite file belongs to. In that case, sqlite.openConnection
-    // will throw. Thus we retey for some time to see if lock is removed.
+    // will throw. Thus we retry for some time to see if lock is removed.
     while (!connection && retryCount++ < 25) {
       try {
         connection = yield Sqlite.openConnection({ path: path });
@@ -2061,8 +2177,14 @@ var indexedDBHelpers = {
       return this.backToChild("getValuesForHost", {objectStores: objectStores});
     }
     // Get either all entries from the object store, or a particular id
-    let result = yield this.getObjectStoreData(host, principal, db2,
-      objectStore, id, options.index, options.size);
+    let storage = hostVsStores.get(host).get(db2).storage;
+    let result = yield this.getObjectStoreData(host, principal, db2, storage, {
+      objectStore: objectStore,
+      id: id,
+      index: options.index,
+      offset: 0,
+      size: options.size
+    });
     return this.backToChild("getValuesForHost", {result: result});
   }),
 
@@ -2076,23 +2198,27 @@ var indexedDBHelpers = {
    *        The principal of the given document.
    * @param {string} dbName
    *        The name of the indexed db from the above host.
-   * @param {string} objectStore
-   *        The name of the object store from the above db.
-   * @param {string} id
-   *        id of the requested entry from the above object store.
-   *        null if all entries from the above object store are requested.
-   * @param {string} index
-   *        name of the IDBIndex to be iterated on while fetching entries.
-   *        null or "name" if no index is to be iterated.
-   * @param {number} offset
-   *        ofsset of the entries to be fetched.
-   * @param {number} size
-   *        The intended size of the entries to be fetched.
+   * @param {String} storage
+   *        Storage type, either "temporary", "default" or "persistent".
+   * @param {Object} requestOptions
+   *        An object in the following format:
+   *        {
+   *          objectStore: The name of the object store from the above db,
+   *          id:          Id of the requested entry from the above object
+   *                       store. null if all entries from the above object
+   *                       store are requested,
+   *          index:       Name of the IDBIndex to be iterated on while fetching
+   *                       entries. null or "name" if no index is to be
+   *                       iterated,
+   *          offset:      offset of the entries to be fetched,
+   *          size:        The intended size of the entries to be fetched
+   *        }
    */
-  getObjectStoreData(host, principal, dbName, objectStore, id, index,
-                     offset, size) {
-    let request = this.openWithPrincipal(principal, dbName);
+  getObjectStoreData(host, principal, dbName, storage, requestOptions) {
+    let {name} = this.splitNameAndStorage(dbName);
+    let request = this.openWithPrincipal(principal, name, storage);
     let success = promise.defer();
+    let {objectStore, id, index, offset, size} = requestOptions;
     let data = [];
     let db;
 
@@ -2194,8 +2320,12 @@ var indexedDBHelpers = {
 
     switch (msg.json.method) {
       case "getDBMetaData": {
-        let [host, principal, name] = args;
-        return indexedDBHelpers.getDBMetaData(host, principal, name);
+        let [host, principal, name, storage] = args;
+        return indexedDBHelpers.getDBMetaData(host, principal, name, storage);
+      }
+      case "splitNameAndStorage": {
+        let [name] = args;
+        return indexedDBHelpers.splitNameAndStorage(name);
       }
       case "getDBNamesForHost": {
         let [host] = args;
@@ -2207,8 +2337,8 @@ var indexedDBHelpers = {
                                                  hostVsStores, principal);
       }
       case "removeDB": {
-        let [host, principal, name] = args;
-        return indexedDBHelpers.removeDB(host, principal, name);
+        let [host, principal, dbName] = args;
+        return indexedDBHelpers.removeDB(host, principal, dbName);
       }
       case "removeDBRecord": {
         let [host, principal, db, store, id] = args;
diff --git a/devtools/server/tests/browser/browser_storage_listings.js b/devtools/server/tests/browser/browser_storage_listings.js
index 2e4bd00a4..15e5ccd50 100644
--- a/devtools/server/tests/browser/browser_storage_listings.js
+++ b/devtools/server/tests/browser/browser_storage_listings.js
@@ -121,24 +121,24 @@ const storeMap = {
 const IDBValues = {
   listStoresResponse: {
     "http://test1.example.org": [
-      ["idb1", "obj1"], ["idb1", "obj2"], ["idb2", "obj3"]
+      ["idb1 (default)", "obj1"], ["idb1 (default)", "obj2"], ["idb2 (default)", "obj3"]
     ],
     "http://sectest1.example.org": [
     ],
     "https://sectest1.example.org": [
-      ["idb-s1", "obj-s1"], ["idb-s2", "obj-s2"]
+      ["idb-s1 (default)", "obj-s1"], ["idb-s2 (default)", "obj-s2"]
     ]
   },
-  dbDetails : {
+  dbDetails: {
     "http://test1.example.org": [
       {
-        db: "idb1",
+        db: "idb1 (default)",
         origin: "http://test1.example.org",
         version: 1,
         objectStores: 2
       },
       {
-        db: "idb2",
+        db: "idb2 (default)",
         origin: "http://test1.example.org",
         version: 1,
         objectStores: 1
@@ -148,13 +148,13 @@ const IDBValues = {
     ],
     "https://sectest1.example.org": [
       {
-        db: "idb-s1",
+        db: "idb-s1 (default)",
         origin: "https://sectest1.example.org",
         version: 1,
         objectStores: 1
       },
       {
-        db: "idb-s2",
+        db: "idb-s2 (default)",
         origin: "https://sectest1.example.org",
         version: 1,
         objectStores: 1
@@ -163,7 +163,7 @@ const IDBValues = {
   },
   objectStoreDetails: {
     "http://test1.example.org": {
-      idb1: [
+      "idb1 (default)": [
         {
           objectStore: "obj1",
           keyPath: "id",
@@ -190,7 +190,7 @@ const IDBValues = {
           indexes: []
         }
       ],
-      idb2: [
+      "idb2 (default)": [
         {
           objectStore: "obj3",
           keyPath: "id3",
@@ -208,7 +208,7 @@ const IDBValues = {
     },
     "http://sectest1.example.org" : {},
     "https://sectest1.example.org": {
-      "idb-s1": [
+      "idb-s1 (default)": [
         {
           objectStore: "obj-s1",
           keyPath: "id",
@@ -216,7 +216,7 @@ const IDBValues = {
           indexes: []
         },
       ],
-      "idb-s2": [
+      "idb-s2 (default)": [
         {
           objectStore: "obj-s2",
           keyPath: "id3",
@@ -236,7 +236,7 @@ const IDBValues = {
   },
   entries: {
     "http://test1.example.org": {
-      "idb1#obj1": [
+      "idb1 (default)#obj1": [
         {
           name: 1,
           value: {
@@ -262,7 +262,7 @@ const IDBValues = {
           }
         }
       ],
-      "idb1#obj2": [
+      "idb1 (default)#obj2": [
         {
           name: 1,
           value: {
@@ -273,11 +273,11 @@ const IDBValues = {
           }
         }
       ],
-      "idb2#obj3": []
+      "idb2 (default)#obj3": []
     },
     "http://sectest1.example.org" : {},
     "https://sectest1.example.org": {
-      "idb-s1#obj-s1": [
+      "idb-s1 (default)#obj-s1": [
         {
           name: 6,
           value: {
@@ -295,7 +295,7 @@ const IDBValues = {
           }
         }
       ],
-      "idb-s2#obj-s2": [
+      "idb-s2 (default)#obj-s2": [
         {
           name: 13,
           value: {