diff options
Diffstat (limited to 'services/sync/modules/engines')
| -rw-r--r-- | services/sync/modules/engines/addons.js | 127 | ||||
| -rw-r--r-- | services/sync/modules/engines/bookmarks.js | 1762 | ||||
| -rw-r--r-- | services/sync/modules/engines/clients.js | 442 | ||||
| -rw-r--r-- | services/sync/modules/engines/forms.js | 91 | ||||
| -rw-r--r-- | services/sync/modules/engines/history.js | 57 | ||||
| -rw-r--r-- | services/sync/modules/engines/passwords.js | 84 | ||||
| -rw-r--r-- | services/sync/modules/engines/prefs.js | 75 | ||||
| -rw-r--r-- | services/sync/modules/engines/tabs.js | 65 |
8 files changed, 1137 insertions, 1566 deletions
diff --git a/services/sync/modules/engines/addons.js b/services/sync/modules/engines/addons.js index 01dab58d1..3081e3e87 100644 --- a/services/sync/modules/engines/addons.js +++ b/services/sync/modules/engines/addons.js @@ -25,13 +25,10 @@ * * Synchronization is influenced by the following preferences: * + * - services.sync.addons.ignoreRepositoryChecking * - services.sync.addons.ignoreUserEnabledChanges * - services.sync.addons.trustedSourceHostnames * - * and also influenced by whether addons have repository caching enabled and - * whether they allow installation of addons from insecure options (both of - * which are themselves influenced by the "extensions." pref branch) - * * See the documentation in services-sync.js for the behavior of these prefs. */ "use strict"; @@ -44,7 +41,6 @@ Cu.import("resource://services-sync/engines.js"); Cu.import("resource://services-sync/record.js"); Cu.import("resource://services-sync/util.js"); Cu.import("resource://services-sync/constants.js"); -Cu.import("resource://services-sync/collection_validator.js"); Cu.import("resource://services-common/async.js"); Cu.import("resource://gre/modules/Preferences.jsm"); @@ -54,7 +50,7 @@ XPCOMUtils.defineLazyModuleGetter(this, "AddonManager", XPCOMUtils.defineLazyModuleGetter(this, "AddonRepository", "resource://gre/modules/addons/AddonRepository.jsm"); -this.EXPORTED_SYMBOLS = ["AddonsEngine", "AddonValidator"]; +this.EXPORTED_SYMBOLS = ["AddonsEngine"]; // 7 days in milliseconds. const PRUNE_ADDON_CHANGES_THRESHOLD = 60 * 60 * 24 * 7 * 1000; @@ -177,7 +173,7 @@ AddonsEngine.prototype = { continue; } - if (!this.isAddonSyncable(addons[id])) { + if (!this._store.isAddonSyncable(addons[id])) { continue; } @@ -235,10 +231,6 @@ AddonsEngine.prototype = { let cb = Async.makeSpinningCallback(); this._reconciler.refreshGlobalState(cb); cb.wait(); - }, - - isAddonSyncable(addon, ignoreRepoCheck) { - return this._store.isAddonSyncable(addon, ignoreRepoCheck); } }; @@ -286,14 +278,6 @@ AddonsStore.prototype = { } } - // Ignore incoming records for which an existing non-syncable addon - // exists. - let existingMeta = this.reconciler.addons[record.addonID]; - if (existingMeta && !this.isAddonSyncable(existingMeta)) { - this._log.info("Ignoring incoming record for an existing but non-syncable addon", record.addonID); - return; - } - Store.prototype.applyIncoming.call(this, record); }, @@ -314,14 +298,6 @@ AddonsStore.prototype = { // engine and the record will try to be applied later. let results = cb.wait(); - if (results.skipped.includes(record.addonID)) { - this._log.info("Add-on skipped: " + record.addonID); - // Just early-return for skipped addons - we don't want to arrange to - // try again next time because the condition that caused up to skip - // will remain true for this addon forever. - return; - } - let addon; for (let a of results.addons) { if (a.id == record.addonID) { @@ -499,7 +475,7 @@ AddonsStore.prototype = { } this._log.info("Uninstalling add-on as part of wipe: " + addon.id); - Utils.catch.call(this, () => addon.uninstall())(); + Utils.catch(addon.uninstall)(); } }, @@ -538,22 +514,16 @@ AddonsStore.prototype = { * * @param addon * Addon instance - * @param ignoreRepoCheck - * Should we skip checking the Addons repository (primarially useful - * for testing and validation). * @return Boolean indicating whether it is appropriate for Sync */ - isAddonSyncable: function isAddonSyncable(addon, ignoreRepoCheck = false) { + isAddonSyncable: function isAddonSyncable(addon) { // Currently, we limit syncable add-ons to those that are: // 1) In a well-defined set of types // 2) Installed in the current profile // 3) Not installed by a foreign entity (i.e. installed by the app) // since they act like global extensions. // 4) Is not a hotfix. - // 5) The addons XPIProvider doesn't veto it (i.e not being installed in - // the profile directory, or any other reasons it says the addon can't - // be synced) - // 6) Are installed from AMO + // 5) Are installed from AMO // We could represent the test as a complex boolean expression. We go the // verbose route so the failure reason is logged. @@ -573,12 +543,6 @@ AddonsStore.prototype = { return false; } - // If the addon manager says it's not syncable, we skip it. - if (!addon.isSyncable) { - this._log.debug(addon.id + " not syncable: vetoed by the addon manager."); - return false; - } - // This may be too aggressive. If an add-on is downloaded from AMO and // manually placed in the profile directory, foreignInstall will be set. // Arguably, that add-on should be syncable. @@ -589,20 +553,15 @@ AddonsStore.prototype = { } // Ignore hotfix extensions (bug 741670). The pref may not be defined. - // XXX - note that addon.isSyncable will be false for hotfix addons, so - // this check isn't strictly necessary - except for Sync tests which aren't - // setup to create a "real" hotfix addon. This can be removed once those - // tests are fixed (but keeping it doesn't hurt either) if (this._extensionsPrefs.get("hotfix.id", null) == addon.id) { this._log.debug(addon.id + " not syncable: is a hotfix."); return false; } - // If the AddonRepository's cache isn't enabled (which it typically isn't - // in tests), getCachedAddonByID always returns null - so skip the check - // in that case. We also provide a way to specifically opt-out of the check - // even if the cache is enabled, which is used by the validators. - if (ignoreRepoCheck || !AddonRepository.cacheEnabled) { + // We provide a back door to skip the repository checking of an add-on. + // This is utilized by the tests to make testing easier. Users could enable + // this, but it would sacrifice security. + if (Svc.Prefs.get("addons.ignoreRepositoryChecking", false)) { return true; } @@ -745,69 +704,3 @@ AddonsTracker.prototype = { this.reconciler.stopListening(); }, }; - -class AddonValidator extends CollectionValidator { - constructor(engine = null) { - super("addons", "id", [ - "addonID", - "enabled", - "applicationID", - "source" - ]); - this.engine = engine; - } - - getClientItems() { - return Promise.all([ - new Promise(resolve => - AddonManager.getAllAddons(resolve)), - new Promise(resolve => - AddonManager.getAddonsWithOperationsByTypes(["extension", "theme"], resolve)), - ]).then(([installed, addonsWithPendingOperation]) => { - // Addons pending install won't be in the first list, but addons pending - // uninstall/enable/disable will be in both lists. - let all = new Map(installed.map(addon => [addon.id, addon])); - for (let addon of addonsWithPendingOperation) { - all.set(addon.id, addon); - } - // Convert to an array since Map.prototype.values returns an iterable - return [...all.values()]; - }); - } - - normalizeClientItem(item) { - let enabled = !item.userDisabled; - if (item.pendingOperations & AddonManager.PENDING_ENABLE) { - enabled = true; - } else if (item.pendingOperations & AddonManager.PENDING_DISABLE) { - enabled = false; - } - return { - enabled, - id: item.syncGUID, - addonID: item.id, - applicationID: Services.appinfo.ID, - source: "amo", // check item.foreignInstall? - original: item - }; - } - - normalizeServerItem(item) { - let guid = this.engine._findDupe(item); - if (guid) { - item.id = guid; - } - return item; - } - - clientUnderstands(item) { - return item.applicationID === Services.appinfo.ID; - } - - syncedByClient(item) { - return !item.original.hidden && - !item.original.isSystem && - !(item.original.pendingOperations & AddonManager.PENDING_UNINSTALL) && - this.engine.isAddonSyncable(item.original, true); - } -} diff --git a/services/sync/modules/engines/bookmarks.js b/services/sync/modules/engines/bookmarks.js index 76a198a8b..41283c06d 100644 --- a/services/sync/modules/engines/bookmarks.js +++ b/services/sync/modules/engines/bookmarks.js @@ -11,7 +11,6 @@ var Ci = Components.interfaces; var Cu = Components.utils; Cu.import("resource://gre/modules/PlacesUtils.jsm"); -Cu.import("resource://gre/modules/PlacesSyncUtils.jsm"); Cu.import("resource://gre/modules/XPCOMUtils.jsm"); Cu.import("resource://services-common/async.js"); Cu.import("resource://services-sync/constants.js"); @@ -20,57 +19,21 @@ Cu.import("resource://services-sync/record.js"); Cu.import("resource://services-sync/util.js"); Cu.import("resource://gre/modules/Task.jsm"); Cu.import("resource://gre/modules/PlacesBackups.jsm"); -XPCOMUtils.defineLazyModuleGetter(this, "BookmarkValidator", - "resource://services-sync/bookmark_validator.js"); -XPCOMUtils.defineLazyGetter(this, "PlacesBundle", () => { - let bundleService = Cc["@mozilla.org/intl/stringbundle;1"] - .getService(Ci.nsIStringBundleService); - return bundleService.createBundle("chrome://places/locale/places.properties"); -}); - -const ANNOS_TO_TRACK = [PlacesSyncUtils.bookmarks.DESCRIPTION_ANNO, - PlacesSyncUtils.bookmarks.SIDEBAR_ANNO, + +const ALLBOOKMARKS_ANNO = "AllBookmarks"; +const DESCRIPTION_ANNO = "bookmarkProperties/description"; +const SIDEBAR_ANNO = "bookmarkProperties/loadInSidebar"; +const MOBILEROOT_ANNO = "mobile/bookmarksRoot"; +const MOBILE_ANNO = "MobileBookmarks"; +const EXCLUDEBACKUP_ANNO = "places/excludeFromBackup"; +const SMART_BOOKMARKS_ANNO = "Places/SmartBookmark"; +const PARENT_ANNO = "sync/parent"; +const ORGANIZERQUERY_ANNO = "PlacesOrganizer/OrganizerQuery"; +const ANNOS_TO_TRACK = [DESCRIPTION_ANNO, SIDEBAR_ANNO, PlacesUtils.LMANNO_FEEDURI, PlacesUtils.LMANNO_SITEURI]; const SERVICE_NOT_SUPPORTED = "Service not supported on this platform"; const FOLDER_SORTINDEX = 1000000; -const { - SOURCE_SYNC, - SOURCE_IMPORT, - SOURCE_IMPORT_REPLACE, -} = Ci.nsINavBookmarksService; - -const SQLITE_MAX_VARIABLE_NUMBER = 999; - -const ORGANIZERQUERY_ANNO = "PlacesOrganizer/OrganizerQuery"; -const ALLBOOKMARKS_ANNO = "AllBookmarks"; -const MOBILE_ANNO = "MobileBookmarks"; - -// The tracker ignores changes made by bookmark import and restore, and -// changes made by Sync. We don't need to exclude `SOURCE_IMPORT`, but both -// import and restore fire `bookmarks-restore-*` observer notifications, and -// the tracker doesn't currently distinguish between the two. -const IGNORED_SOURCES = [SOURCE_SYNC, SOURCE_IMPORT, SOURCE_IMPORT_REPLACE]; - -// Returns the constructor for a bookmark record type. -function getTypeObject(type) { - switch (type) { - case "bookmark": - case "microsummary": - return Bookmark; - case "query": - return BookmarkQuery; - case "folder": - return BookmarkFolder; - case "livemark": - return Livemark; - case "separator": - return BookmarkSeparator; - case "item": - return PlacesItem; - } - return null; -} this.PlacesItem = function PlacesItem(collection, id, type) { CryptoWrapper.call(this, collection, id); @@ -89,32 +52,26 @@ PlacesItem.prototype = { }, getTypeObject: function PlacesItem_getTypeObject(type) { - let recordObj = getTypeObject(type); - if (!recordObj) { - throw new Error("Unknown places item object type: " + type); + switch (type) { + case "bookmark": + case "microsummary": + return Bookmark; + case "query": + return BookmarkQuery; + case "folder": + return BookmarkFolder; + case "livemark": + return Livemark; + case "separator": + return BookmarkSeparator; + case "item": + return PlacesItem; } - return recordObj; + throw "Unknown places item object type: " + type; }, __proto__: CryptoWrapper.prototype, _logName: "Sync.Record.PlacesItem", - - // Converts the record to a Sync bookmark object that can be passed to - // `PlacesSyncUtils.bookmarks.{insert, update}`. - toSyncBookmark() { - return { - kind: this.type, - syncId: this.id, - parentSyncId: this.parentid, - }; - }, - - // Populates the record from a Sync bookmark object returned from - // `PlacesSyncUtils.bookmarks.fetch`. - fromSyncBookmark(item) { - this.parentid = item.parentSyncId; - this.parentName = item.parentTitle; - }, }; Utils.deferGetSet(PlacesItem, @@ -127,27 +84,6 @@ this.Bookmark = function Bookmark(collection, id, type) { Bookmark.prototype = { __proto__: PlacesItem.prototype, _logName: "Sync.Record.Bookmark", - - toSyncBookmark() { - let info = PlacesItem.prototype.toSyncBookmark.call(this); - info.title = this.title; - info.url = this.bmkUri; - info.description = this.description; - info.loadInSidebar = this.loadInSidebar; - info.tags = this.tags; - info.keyword = this.keyword; - return info; - }, - - fromSyncBookmark(item) { - PlacesItem.prototype.fromSyncBookmark.call(this, item); - this.title = item.title; - this.bmkUri = item.url.href; - this.description = item.description; - this.loadInSidebar = item.loadInSidebar; - this.tags = item.tags; - this.keyword = item.keyword; - }, }; Utils.deferGetSet(Bookmark, @@ -161,19 +97,6 @@ this.BookmarkQuery = function BookmarkQuery(collection, id) { BookmarkQuery.prototype = { __proto__: Bookmark.prototype, _logName: "Sync.Record.BookmarkQuery", - - toSyncBookmark() { - let info = Bookmark.prototype.toSyncBookmark.call(this); - info.folder = this.folderName; - info.query = this.queryId; - return info; - }, - - fromSyncBookmark(item) { - Bookmark.prototype.fromSyncBookmark.call(this, item); - this.folderName = item.folder; - this.queryId = item.query; - }, }; Utils.deferGetSet(BookmarkQuery, @@ -186,20 +109,6 @@ this.BookmarkFolder = function BookmarkFolder(collection, id, type) { BookmarkFolder.prototype = { __proto__: PlacesItem.prototype, _logName: "Sync.Record.Folder", - - toSyncBookmark() { - let info = PlacesItem.prototype.toSyncBookmark.call(this); - info.description = this.description; - info.title = this.title; - return info; - }, - - fromSyncBookmark(item) { - PlacesItem.prototype.fromSyncBookmark.call(this, item); - this.title = item.title; - this.description = item.description; - this.children = item.childSyncIds; - }, }; Utils.deferGetSet(BookmarkFolder, "cleartext", ["description", "title", @@ -211,21 +120,6 @@ this.Livemark = function Livemark(collection, id) { Livemark.prototype = { __proto__: BookmarkFolder.prototype, _logName: "Sync.Record.Livemark", - - toSyncBookmark() { - let info = BookmarkFolder.prototype.toSyncBookmark.call(this); - info.feed = this.feedUri; - info.site = this.siteUri; - return info; - }, - - fromSyncBookmark(item) { - BookmarkFolder.prototype.fromSyncBookmark.call(this, item); - this.feedUri = item.feed.href; - if (item.site) { - this.siteUri = item.site.href; - } - }, }; Utils.deferGetSet(Livemark, "cleartext", ["siteUri", "feedUri"]); @@ -236,15 +130,81 @@ this.BookmarkSeparator = function BookmarkSeparator(collection, id) { BookmarkSeparator.prototype = { __proto__: PlacesItem.prototype, _logName: "Sync.Record.Separator", - - fromSyncBookmark(item) { - PlacesItem.prototype.fromSyncBookmark.call(this, item); - this.pos = item.index; - }, }; Utils.deferGetSet(BookmarkSeparator, "cleartext", "pos"); + +let kSpecialIds = { + + // Special IDs. Note that mobile can attempt to create a record on + // dereference; special accessors are provided to prevent recursion within + // observers. + guids: ["menu", "places", "tags", "toolbar", "unfiled", "mobile"], + + // Create the special mobile folder to store mobile bookmarks. + createMobileRoot: function createMobileRoot() { + let root = PlacesUtils.placesRootId; + let mRoot = PlacesUtils.bookmarks.createFolder(root, "mobile", -1); + PlacesUtils.annotations.setItemAnnotation( + mRoot, MOBILEROOT_ANNO, 1, 0, PlacesUtils.annotations.EXPIRE_NEVER); + PlacesUtils.annotations.setItemAnnotation( + mRoot, EXCLUDEBACKUP_ANNO, 1, 0, PlacesUtils.annotations.EXPIRE_NEVER); + return mRoot; + }, + + findMobileRoot: function findMobileRoot(create) { + // Use the (one) mobile root if it already exists. + let root = PlacesUtils.annotations.getItemsWithAnnotation(MOBILEROOT_ANNO, {}); + if (root.length != 0) + return root[0]; + + if (create) + return this.createMobileRoot(); + + return null; + }, + + // Accessors for IDs. + isSpecialGUID: function isSpecialGUID(g) { + return this.guids.indexOf(g) != -1; + }, + + specialIdForGUID: function specialIdForGUID(guid, create) { + if (guid == "mobile") { + return this.findMobileRoot(create); + } + return this[guid]; + }, + + // Don't bother creating mobile: if it doesn't exist, this ID can't be it! + specialGUIDForId: function specialGUIDForId(id) { + for each (let guid in this.guids) + if (this.specialIdForGUID(guid, false) == id) + return guid; + return null; + }, + + get menu() { + return PlacesUtils.bookmarksMenuFolderId; + }, + get places() { + return PlacesUtils.placesRootId; + }, + get tags() { + return PlacesUtils.tagsFolderId; + }, + get toolbar() { + return PlacesUtils.toolbarFolderId; + }, + get unfiled() { + return PlacesUtils.unfiledBookmarksFolderId; + }, + get mobile() { + return this.findMobileRoot(true); + }, +}; + this.BookmarksEngine = function BookmarksEngine(service) { SyncEngine.call(this, "Bookmarks", service); } @@ -257,103 +217,68 @@ BookmarksEngine.prototype = { _defaultSort: "index", syncPriority: 4, - allowSkippedRecord: false, - - // A diagnostic helper to get the string value for a bookmark's URL given - // its ID. Always returns a string - on error will return a string in the - // form of "<description of error>" as this is purely for, eg, logging. - // (This means hitting the DB directly and we don't bother using a cached - // statement - we should rarely hit this.) - _getStringUrlForId(id) { - let url; - try { - let stmt = this._store._getStmt(` - SELECT h.url - FROM moz_places h - JOIN moz_bookmarks b ON h.id = b.fk - WHERE b.id = :id`); - stmt.params.id = id; - let rows = Async.querySpinningly(stmt, ["url"]); - url = rows.length == 0 ? "<not found>" : rows[0].url; - } catch (ex) { - if (Async.isShutdownException(ex)) { - throw ex; - } - if (ex instanceof Ci.mozIStorageError) { - url = `<failed: Storage error: ${ex.message} (${ex.result})>`; - } else { - url = `<failed: ${ex.toString()}>`; - } - } - return url; - }, - _guidMapFailed: false, - _buildGUIDMap: function _buildGUIDMap() { - let store = this._store; - let guidMap = {}; - let tree = Async.promiseSpinningly(PlacesUtils.promiseBookmarksTree("", { - includeItemIds: true - })); - function* walkBookmarksTree(tree, parent=null) { - if (tree) { - // Skip root node - if (parent) { - yield [tree, parent]; + _sync: function _sync() { + let engine = this; + let batchEx = null; + + // Try running sync in batch mode + PlacesUtils.bookmarks.runInBatchMode({ + runBatched: function wrappedSync() { + try { + SyncEngine.prototype._sync.call(engine); } - if (tree.children) { - for (let child of tree.children) { - store._sleep(0); // avoid jank while looping. - yield* walkBookmarksTree(child, tree); - } + catch(ex) { + batchEx = ex; } } - } + }, null); - function* walkBookmarksRoots(tree, rootIDs) { - for (let id of rootIDs) { - let bookmarkRoot = tree.children.find(child => child.id === id); - if (bookmarkRoot === null) { - continue; - } - yield* walkBookmarksTree(bookmarkRoot, tree); - } + // Expose the exception if something inside the batch failed + if (batchEx != null) { + throw batchEx; } + }, - let rootsToWalk = getChangeRootIds(); - - for (let [node, parent] of walkBookmarksRoots(tree, rootsToWalk)) { - let {guid, id, type: placeType} = node; - guid = PlacesSyncUtils.bookmarks.guidToSyncId(guid); + _guidMapFailed: false, + _buildGUIDMap: function _buildGUIDMap() { + let guidMap = {}; + for (let guid in this._store.getAllIDs()) { + // Figure out with which key to store the mapping. let key; - switch (placeType) { - case PlacesUtils.TYPE_X_MOZ_PLACE: - // Bookmark - let query = null; - if (node.annos && node.uri.startsWith("place:")) { - query = node.annos.find(({name}) => - name === PlacesSyncUtils.bookmarks.SMART_BOOKMARKS_ANNO); - } - if (query && query.value) { - key = "q" + query.value; - } else { - key = "b" + node.uri + ":" + (node.title || ""); - } + let id = this._store.idForGUID(guid); + switch (PlacesUtils.bookmarks.getItemType(id)) { + case PlacesUtils.bookmarks.TYPE_BOOKMARK: + + // Smart bookmarks map to their annotation value. + let queryId; + try { + queryId = PlacesUtils.annotations.getItemAnnotation( + id, SMART_BOOKMARKS_ANNO); + } catch(ex) {} + + if (queryId) + key = "q" + queryId; + else + key = "b" + PlacesUtils.bookmarks.getBookmarkURI(id).spec + ":" + + PlacesUtils.bookmarks.getItemTitle(id); break; - case PlacesUtils.TYPE_X_MOZ_PLACE_CONTAINER: - // Folder - key = "f" + (node.title || ""); + case PlacesUtils.bookmarks.TYPE_FOLDER: + key = "f" + PlacesUtils.bookmarks.getItemTitle(id); break; - case PlacesUtils.TYPE_X_MOZ_PLACE_SEPARATOR: - // Separator - key = "s" + node.index; + case PlacesUtils.bookmarks.TYPE_SEPARATOR: + key = "s" + PlacesUtils.bookmarks.getItemIndex(id); break; default: - this._log.error("Unknown place type: '"+placeType+"'"); continue; } - let parentName = parent.title || ""; + // The mapping is on a per parent-folder-name basis. + let parent = PlacesUtils.bookmarks.getFolderIdForItem(id); + if (parent <= 0) + continue; + + let parentName = PlacesUtils.bookmarks.getItemTitle(parent); if (guidMap[parentName] == null) guidMap[parentName] = {}; @@ -381,17 +306,17 @@ BookmarksEngine.prototype = { // hack should get them to dupe correctly. if (item.queryId) { key = "q" + item.queryId; - altKey = "b" + item.bmkUri + ":" + (item.title || ""); + altKey = "b" + item.bmkUri + ":" + item.title; break; } // No queryID? Fall through to the regular bookmark case. case "bookmark": case "microsummary": - key = "b" + item.bmkUri + ":" + (item.title || ""); + key = "b" + item.bmkUri + ":" + item.title; break; case "folder": case "livemark": - key = "f" + (item.title || ""); + key = "f" + item.title; break; case "separator": key = "s" + item.pos; @@ -405,22 +330,21 @@ BookmarksEngine.prototype = { let guidMap = this._guidMap; // Give the GUID if we have the matching pair. - let parentName = item.parentName || ""; - this._log.trace("Finding mapping: " + parentName + ", " + key); - let parent = guidMap[parentName]; - + this._log.trace("Finding mapping: " + item.parentName + ", " + key); + let parent = guidMap[item.parentName]; + if (!parent) { this._log.trace("No parent => no dupe."); return undefined; } - + let dupe = parent[key]; - + if (dupe) { this._log.trace("Mapped dupe: " + dupe); return dupe; } - + if (altKey) { dupe = parent[altKey]; if (dupe) { @@ -428,7 +352,7 @@ BookmarksEngine.prototype = { return dupe; } } - + this._log.trace("No dupe found for key " + key + "/" + altKey + "."); return undefined; }, @@ -437,7 +361,7 @@ BookmarksEngine.prototype = { SyncEngine.prototype._syncStartup.call(this); let cb = Async.makeSpinningCallback(); - Task.spawn(function* () { + Task.spawn(function() { // For first-syncs, make a backup for the user to restore if (this.lastSync == 0) { this._log.debug("Bookmarks backup starting."); @@ -449,7 +373,7 @@ BookmarksEngine.prototype = { // Failure to create a backup is somewhat bad, but probably not bad // enough to prevent syncing of bookmarks - so just log the error and // continue. - this._log.warn("Error while backing up bookmarks, but continuing with sync", ex); + this._log.warn("Got exception backing up bookmarks, but continuing with sync.", ex); cb(); } ); @@ -464,10 +388,8 @@ BookmarksEngine.prototype = { try { guidMap = this._buildGUIDMap(); } catch (ex) { - if (Async.isShutdownException(ex)) { - throw ex; - } - this._log.warn("Error while building GUID map, skipping all other incoming items", ex); + this._log.warn("Got exception building GUID map." + + " Skipping all other incoming items.", ex); throw {code: Engine.prototype.eEngineAbortApplyIncoming, cause: ex}; } @@ -476,71 +398,17 @@ BookmarksEngine.prototype = { }); this._store._childrenToOrder = {}; - this._store.clearPendingDeletions(); - }, - - _deletePending() { - // Delete pending items -- See the comment above BookmarkStore's deletePending - let newlyModified = Async.promiseSpinningly(this._store.deletePending()); - let now = this._tracker._now(); - this._log.debug("Deleted pending items", newlyModified); - for (let modifiedSyncID of newlyModified) { - if (!this._modified.has(modifiedSyncID)) { - this._modified.set(modifiedSyncID, { timestamp: now, deleted: false }); - } - } - }, - - // We avoid reviving folders since reviving them properly would require - // reviving their children as well. Unfortunately, this is the wrong choice - // in the case of a bookmark restore where wipeServer failed -- if the - // server has the folder as deleted, we *would* want to reupload this folder. - // This is mitigated by the fact that we move any undeleted children to the - // grandparent when deleting the parent. - _shouldReviveRemotelyDeletedRecord(item) { - let kind = Async.promiseSpinningly( - PlacesSyncUtils.bookmarks.getKindForSyncId(item.id)); - if (kind === PlacesSyncUtils.bookmarks.KINDS.FOLDER) { - return false; - } - - // In addition to preventing the deletion of this record (handled by the caller), - // we need to mark the parent of this record for uploading next sync, in order - // to ensure its children array is accurate. - let modifiedTimestamp = this._modified.getModifiedTimestamp(item.id); - if (!modifiedTimestamp) { - // We only expect this to be called with items locally modified, so - // something strange is going on - play it safe and don't revive it. - this._log.error("_shouldReviveRemotelyDeletedRecord called on unmodified item: " + item.id); - return false; - } - - let localID = this._store.idForGUID(item.id); - let localParentID = PlacesUtils.bookmarks.getFolderIdForItem(localID); - let localParentSyncID = this._store.GUIDForId(localParentID); - - this._log.trace(`Reviving item "${item.id}" and marking parent ${localParentSyncID} as modified.`); - - if (!this._modified.has(localParentSyncID)) { - this._modified.set(localParentSyncID, { - timestamp: modifiedTimestamp, - deleted: false - }); - } - return true }, _processIncoming: function (newitems) { try { SyncEngine.prototype._processIncoming.call(this, newitems); } finally { - try { - this._deletePending(); - } finally { - // Reorder children. - this._store._orderChildren(); - delete this._store._childrenToOrder; - } + // Reorder children. + this._tracker.ignoreAll = true; + this._store._orderChildren(); + this._tracker.ignoreAll = false; + delete this._store._childrenToOrder; } }, @@ -575,154 +443,16 @@ BookmarksEngine.prototype = { } let mapped = this._mapDupe(item); this._log.debug(item.id + " mapped to " + mapped); - // We must return a string, not an object, and the entries in the GUIDMap - // are created via "new String()" making them an object. - return mapped ? mapped.toString() : mapped; - }, - - pullAllChanges() { - return new BookmarksChangeset(this._store.getAllIDs()); - }, - - pullNewChanges() { - let modifiedGUIDs = this._getModifiedGUIDs(); - if (!modifiedGUIDs.length) { - return new BookmarksChangeset(this._tracker.changedIDs); - } - - // We don't use `PlacesUtils.promiseDBConnection` here because - // `getChangedIDs` might be called while we're in a batch, meaning we - // won't see any changes until the batch finishes and the transaction - // commits. - let db = PlacesUtils.history.QueryInterface(Ci.nsPIPlacesDatabase) - .DBConnection; - - // Filter out tags, organizer queries, and other descendants that we're - // not tracking. We chunk `modifiedGUIDs` because SQLite limits the number - // of bound parameters per query. - for (let startIndex = 0; - startIndex < modifiedGUIDs.length; - startIndex += SQLITE_MAX_VARIABLE_NUMBER) { - - let chunkLength = Math.min(SQLITE_MAX_VARIABLE_NUMBER, - modifiedGUIDs.length - startIndex); - - let query = ` - WITH RECURSIVE - modifiedGuids(guid) AS ( - VALUES ${new Array(chunkLength).fill("(?)").join(", ")} - ), - syncedItems(id) AS ( - VALUES ${getChangeRootIds().map(id => `(${id})`).join(", ")} - UNION ALL - SELECT b.id - FROM moz_bookmarks b - JOIN syncedItems s ON b.parent = s.id - ) - SELECT b.guid - FROM modifiedGuids m - JOIN moz_bookmarks b ON b.guid = m.guid - LEFT JOIN syncedItems s ON b.id = s.id - WHERE s.id IS NULL - `; - - let statement = db.createAsyncStatement(query); - try { - for (let i = 0; i < chunkLength; i++) { - statement.bindByIndex(i, modifiedGUIDs[startIndex + i]); - } - let results = Async.querySpinningly(statement, ["guid"]); - for (let { guid } of results) { - let syncID = PlacesSyncUtils.bookmarks.guidToSyncId(guid); - this._tracker.removeChangedID(syncID); - } - } finally { - statement.finalize(); - } - } - - return new BookmarksChangeset(this._tracker.changedIDs); - }, - - // Returns an array of Places GUIDs for all changed items. Ignores deletions, - // which won't exist in the DB and shouldn't be removed from the tracker. - _getModifiedGUIDs() { - let guids = []; - for (let syncID in this._tracker.changedIDs) { - if (this._tracker.changedIDs[syncID].deleted === true) { - // The `===` check also filters out old persisted timestamps, - // which won't have a `deleted` property. - continue; - } - let guid = PlacesSyncUtils.bookmarks.syncIdToGuid(syncID); - guids.push(guid); - } - return guids; - }, - - // Called when _findDupe returns a dupe item and the engine has decided to - // switch the existing item to the new incoming item. - _switchItemToDupe(localDupeGUID, incomingItem) { - // We unconditionally change the item's ID in case the engine knows of - // an item but doesn't expose it through itemExists. If the API - // contract were stronger, this could be changed. - this._log.debug("Switching local ID to incoming: " + localDupeGUID + " -> " + - incomingItem.id); - this._store.changeItemID(localDupeGUID, incomingItem.id); - - // And mark the parent as being modified. Given we de-dupe based on the - // parent *name* it's possible the item having its GUID changed has a - // different parent from the incoming record. - // So we need to find the GUID of the local parent. - let now = this._tracker._now(); - let localID = this._store.idForGUID(incomingItem.id); - let localParentID = PlacesUtils.bookmarks.getFolderIdForItem(localID); - let localParentGUID = this._store.GUIDForId(localParentID); - this._modified.set(localParentGUID, { modified: now, deleted: false }); - - // And we also add the parent as reflected in the incoming record as the - // de-dupe process might have used an existing item in a different folder. - // But only if the parent exists, otherwise we will upload a deleted item - // when it might actually be valid, just unknown to us. Note that this - // scenario will still leave us with inconsistent client and server states; - // the incoming record on the server references a parent that isn't the - // actual parent locally - see bug 1297955. - if (localParentGUID != incomingItem.parentid) { - let remoteParentID = this._store.idForGUID(incomingItem.parentid); - if (remoteParentID > 0) { - // The parent specified in the record does exist, so we are going to - // attempt a move when we come to applying the record. Mark the parent - // as being modified so we will later upload it with the new child - // reference. - this._modified.set(incomingItem.parentid, { modified: now, deleted: false }); - } else { - // We aren't going to do a move as we don't have the parent (yet?). - // When applying the record we will add our special PARENT_ANNO - // annotation, so if it arrives in the future (either this Sync or a - // later one) it will be reparented. - this._log.debug(`Incoming duplicate item ${incomingItem.id} specifies ` + - `non-existing parent ${incomingItem.parentid}`); - } - } - - // The local, duplicate ID is always deleted on the server - but for - // bookmarks it is a logical delete. - // Simply adding this (now non-existing) ID to the tracker is enough. - this._modified.set(localDupeGUID, { modified: now, deleted: true }); - }, - getValidator() { - return new BookmarkValidator(); + return mapped; } }; function BookmarksStore(name, engine) { Store.call(this, name, engine); - this._foldersToDelete = new Set(); - this._atomsToDelete = new Set(); + // Explicitly nullify our references to our cached services so we don't leak Svc.Obs.add("places-shutdown", function() { - for (let query in this._stmts) { - let stmt = this._stmts[query]; + for each (let [query, stmt] in Iterator(this._stmts)) { stmt.finalize(); } this._stmts = {}; @@ -732,12 +462,70 @@ BookmarksStore.prototype = { __proto__: Store.prototype, itemExists: function BStore_itemExists(id) { - return this.idForGUID(id) > 0; + return this.idForGUID(id, true) > 0; }, + + /* + * If the record is a tag query, rewrite it to refer to the local tag ID. + * + * Otherwise, just return. + */ + preprocessTagQuery: function preprocessTagQuery(record) { + if (record.type != "query" || + record.bmkUri == null || + !record.folderName) + return; + + // Yes, this works without chopping off the "place:" prefix. + let uri = record.bmkUri + let queriesRef = {}; + let queryCountRef = {}; + let optionsRef = {}; + PlacesUtils.history.queryStringToQueries(uri, queriesRef, queryCountRef, + optionsRef); + + // We only process tag URIs. + if (optionsRef.value.resultType != optionsRef.value.RESULTS_AS_TAG_CONTENTS) + return; + + // Tag something to ensure that the tag exists. + let tag = record.folderName; + let dummyURI = Utils.makeURI("about:weave#BStore_preprocess"); + PlacesUtils.tagging.tagURI(dummyURI, [tag]); + + // Look for the id of the tag, which might just have been added. + let tags = this._getNode(PlacesUtils.tagsFolderId); + if (!(tags instanceof Ci.nsINavHistoryQueryResultNode)) { + this._log.debug("tags isn't an nsINavHistoryQueryResultNode; aborting."); + return; + } + tags.containerOpen = true; + try { + for (let i = 0; i < tags.childCount; i++) { + let child = tags.getChild(i); + if (child.title == tag) { + // Found the tag, so fix up the query to use the right id. + this._log.debug("Tag query folder: " + tag + " = " + child.itemId); + + this._log.trace("Replacing folders in: " + uri); + for each (let q in queriesRef.value) + q.setFolders([child.itemId], 1); + + record.bmkUri = PlacesUtils.history.queriesToQueryString( + queriesRef.value, queryCountRef.value, optionsRef.value); + return; + } + } + } + finally { + tags.containerOpen = false; + } + }, + applyIncoming: function BStore_applyIncoming(record) { this._log.debug("Applying record " + record.id); - let isSpecial = PlacesSyncUtils.bookmarks.ROOTS.includes(record.id); + let isSpecial = record.id in kSpecialIds; if (record.deleted) { if (isSpecial) { @@ -765,217 +553,548 @@ BookmarksStore.prototype = { return; } + // Preprocess the record before doing the normal apply. + this.preprocessTagQuery(record); + // Figure out the local id of the parent GUID if available let parentGUID = record.parentid; if (!parentGUID) { throw "Record " + record.id + " has invalid parentid: " + parentGUID; } - this._log.debug("Remote parent is " + parentGUID); + this._log.debug("Local parent is " + parentGUID); + + let parentId = this.idForGUID(parentGUID); + if (parentId > 0) { + // Save the parent id for modifying the bookmark later + record._parent = parentId; + record._orphan = false; + this._log.debug("Record " + record.id + " is not an orphan."); + } else { + this._log.trace("Record " + record.id + + " is an orphan: could not find parent " + parentGUID); + record._orphan = true; + } // Do the normal processing of incoming records Store.prototype.applyIncoming.call(this, record); - if (record.type == "folder" && record.children) { - this._childrenToOrder[record.id] = record.children; + // Do some post-processing if we have an item + let itemId = this.idForGUID(record.id); + if (itemId > 0) { + // Move any children that are looking for this folder as a parent + if (record.type == "folder") { + this._reparentOrphans(itemId); + // Reorder children later + if (record.children) + this._childrenToOrder[record.id] = record.children; + } + + // Create an annotation to remember that it needs reparenting. + if (record._orphan) { + PlacesUtils.annotations.setItemAnnotation( + itemId, PARENT_ANNO, parentGUID, 0, + PlacesUtils.annotations.EXPIRE_NEVER); + } + } + }, + + /** + * Find all ids of items that have a given value for an annotation + */ + _findAnnoItems: function BStore__findAnnoItems(anno, val) { + return PlacesUtils.annotations.getItemsWithAnnotation(anno, {}) + .filter(function(id) { + return PlacesUtils.annotations.getItemAnnotation(id, anno) == val; + }); + }, + + /** + * For the provided parent item, attach its children to it + */ + _reparentOrphans: function _reparentOrphans(parentId) { + // Find orphans and reunite with this folder parent + let parentGUID = this.GUIDForId(parentId); + let orphans = this._findAnnoItems(PARENT_ANNO, parentGUID); + + this._log.debug("Reparenting orphans " + orphans + " to " + parentId); + orphans.forEach(function(orphan) { + // Move the orphan to the parent and drop the missing parent annotation + if (this._reparentItem(orphan, parentId)) { + PlacesUtils.annotations.removeItemAnnotation(orphan, PARENT_ANNO); + } + }, this); + }, + + _reparentItem: function _reparentItem(itemId, parentId) { + this._log.trace("Attempting to move item " + itemId + " to new parent " + + parentId); + try { + if (parentId > 0) { + PlacesUtils.bookmarks.moveItem(itemId, parentId, + PlacesUtils.bookmarks.DEFAULT_INDEX); + return true; + } + } catch(ex) { + this._log.debug("Failed to reparent item. ", ex); + } + return false; + }, + + // Turn a record's nsINavBookmarksService constant and other attributes into + // a granular type for comparison. + _recordType: function _recordType(itemId) { + let bms = PlacesUtils.bookmarks; + let type = bms.getItemType(itemId); + + switch (type) { + case bms.TYPE_FOLDER: + if (PlacesUtils.annotations + .itemHasAnnotation(itemId, PlacesUtils.LMANNO_FEEDURI)) { + return "livemark"; + } + return "folder"; + + case bms.TYPE_BOOKMARK: + let bmkUri = bms.getBookmarkURI(itemId).spec; + if (bmkUri.indexOf("place:") == 0) { + return "query"; + } + return "bookmark"; + + case bms.TYPE_SEPARATOR: + return "separator"; + + default: + return null; } }, create: function BStore_create(record) { - let info = record.toSyncBookmark(); - // This can throw if we're inserting an invalid or incomplete bookmark. - // That's fine; the exception will be caught by `applyIncomingBatch` - // without aborting further processing. - let item = Async.promiseSpinningly(PlacesSyncUtils.bookmarks.insert(info)); - if (item) { - this._log.debug(`Created ${item.kind} ${item.syncId} under ${ - item.parentSyncId}`, item); + // Default to unfiled if we don't have the parent yet. + + // Valid parent IDs are all positive integers. Other values -- undefined, + // null, -1 -- all compare false for > 0, so this catches them all. We + // don't just use <= without the !, because undefined and null compare + // false for that, too! + if (!(record._parent > 0)) { + this._log.debug("Parent is " + record._parent + "; reparenting to unfiled."); + record._parent = kSpecialIds.unfiled; + } + + let newId; + switch (record.type) { + case "bookmark": + case "query": + case "microsummary": { + let uri = Utils.makeURI(record.bmkUri); + newId = PlacesUtils.bookmarks.insertBookmark( + record._parent, uri, PlacesUtils.bookmarks.DEFAULT_INDEX, record.title); + this._log.debug("created bookmark " + newId + " under " + record._parent + + " as " + record.title + " " + record.bmkUri); + + // Smart bookmark annotations are strings. + if (record.queryId) { + PlacesUtils.annotations.setItemAnnotation( + newId, SMART_BOOKMARKS_ANNO, record.queryId, 0, + PlacesUtils.annotations.EXPIRE_NEVER); + } + + if (Array.isArray(record.tags)) { + this._tagURI(uri, record.tags); + } + PlacesUtils.bookmarks.setKeywordForBookmark(newId, record.keyword); + if (record.description) { + PlacesUtils.annotations.setItemAnnotation( + newId, DESCRIPTION_ANNO, record.description, 0, + PlacesUtils.annotations.EXPIRE_NEVER); + } + + if (record.loadInSidebar) { + PlacesUtils.annotations.setItemAnnotation( + newId, SIDEBAR_ANNO, true, 0, + PlacesUtils.annotations.EXPIRE_NEVER); + } + + } break; + case "folder": + newId = PlacesUtils.bookmarks.createFolder( + record._parent, record.title, PlacesUtils.bookmarks.DEFAULT_INDEX); + this._log.debug("created folder " + newId + " under " + record._parent + + " as " + record.title); + + if (record.description) { + PlacesUtils.annotations.setItemAnnotation( + newId, DESCRIPTION_ANNO, record.description, 0, + PlacesUtils.annotations.EXPIRE_NEVER); + } + + // record.children will be dealt with in _orderChildren. + break; + case "livemark": + let siteURI = null; + if (!record.feedUri) { + this._log.debug("No feed URI: skipping livemark record " + record.id); + return; + } + if (PlacesUtils.annotations + .itemHasAnnotation(record._parent, PlacesUtils.LMANNO_FEEDURI)) { + this._log.debug("Invalid parent: skipping livemark record " + record.id); + return; + } + + if (record.siteUri != null) + siteURI = Utils.makeURI(record.siteUri); + + // Until this engine can handle asynchronous error reporting, we need to + // detect errors on creation synchronously. + let spinningCb = Async.makeSpinningCallback(); + + let livemarkObj = {title: record.title, + parentId: record._parent, + index: PlacesUtils.bookmarks.DEFAULT_INDEX, + feedURI: Utils.makeURI(record.feedUri), + siteURI: siteURI, + guid: record.id}; + PlacesUtils.livemarks.addLivemark(livemarkObj).then( + aLivemark => { spinningCb(null, [Components.results.NS_OK, aLivemark]) }, + () => { spinningCb(null, [Components.results.NS_ERROR_UNEXPECTED, aLivemark]) } + ); + + let [status, livemark] = spinningCb.wait(); + if (!Components.isSuccessCode(status)) { + throw status; + } + + this._log.debug("Created livemark " + livemark.id + " under " + + livemark.parentId + " as " + livemark.title + + ", " + livemark.siteURI.spec + ", " + + livemark.feedURI.spec + ", GUID " + + livemark.guid); + break; + case "separator": + newId = PlacesUtils.bookmarks.insertSeparator( + record._parent, PlacesUtils.bookmarks.DEFAULT_INDEX); + this._log.debug("created separator " + newId + " under " + record._parent); + break; + case "item": + this._log.debug(" -> got a generic places item.. do nothing?"); + return; + default: + this._log.error("_create: Unknown item type: " + record.type); + return; + } + + if (newId) { + // Livemarks can set the GUID through the API, so there's no need to + // do that here. + this._log.trace("Setting GUID of new item " + newId + " to " + record.id); + this._setGUID(newId, record.id); + } + }, + + // Factored out of `remove` to avoid redundant DB queries when the Places ID + // is already known. + removeById: function removeById(itemId, guid) { + let type = PlacesUtils.bookmarks.getItemType(itemId); + + switch (type) { + case PlacesUtils.bookmarks.TYPE_BOOKMARK: + this._log.debug(" -> removing bookmark " + guid); + PlacesUtils.bookmarks.removeItem(itemId); + break; + case PlacesUtils.bookmarks.TYPE_FOLDER: + this._log.debug(" -> removing folder " + guid); + PlacesUtils.bookmarks.removeItem(itemId); + break; + case PlacesUtils.bookmarks.TYPE_SEPARATOR: + this._log.debug(" -> removing separator " + guid); + PlacesUtils.bookmarks.removeItem(itemId); + break; + default: + this._log.error("remove: Unknown item type: " + type); + break; } }, remove: function BStore_remove(record) { - if (PlacesSyncUtils.bookmarks.isRootSyncID(record.id)) { + if (kSpecialIds.isSpecialGUID(record.id)) { this._log.warn("Refusing to remove special folder " + record.id); return; } - let recordKind = Async.promiseSpinningly( - PlacesSyncUtils.bookmarks.getKindForSyncId(record.id)); - let isFolder = recordKind === PlacesSyncUtils.bookmarks.KINDS.FOLDER; - this._log.trace(`Buffering removal of item "${record.id}" of type "${recordKind}".`); - if (isFolder) { - this._foldersToDelete.add(record.id); - } else { - this._atomsToDelete.add(record.id); + + let itemId = this.idForGUID(record.id); + if (itemId <= 0) { + this._log.debug("Item " + record.id + " already removed"); + return; } + this.removeById(itemId, record.id); }, - update: function BStore_update(record) { - let info = record.toSyncBookmark(); - let item = Async.promiseSpinningly(PlacesSyncUtils.bookmarks.update(info)); - if (item) { - this._log.debug(`Updated ${item.kind} ${item.syncId} under ${ - item.parentSyncId}`, item); - } + _taggableTypes: ["bookmark", "microsummary", "query"], + isTaggable: function isTaggable(recordType) { + return this._taggableTypes.indexOf(recordType) != -1; }, - _orderChildren: function _orderChildren() { - let promises = Object.keys(this._childrenToOrder).map(syncID => { - let children = this._childrenToOrder[syncID]; - return PlacesSyncUtils.bookmarks.order(syncID, children).catch(ex => { - this._log.debug(`Could not order children for ${syncID}`, ex); - }); - }); - Async.promiseSpinningly(Promise.all(promises)); - }, - - // There's some complexity here around pending deletions. Our goals: - // - // - Don't delete any bookmarks a user has created but not explicitly deleted - // (This includes any bookmark that was not a child of the folder at the - // time the deletion was recorded, and also bookmarks restored from a backup). - // - Don't undelete any bookmark without ensuring the server structure - // includes it (see `BookmarkEngine.prototype._shouldReviveRemotelyDeletedRecord`) - // - // This leads the following approach: - // - // - Additions, moves, and updates are processed before deletions. - // - To do this, all deletion operations are buffered during a sync. Folders - // we plan on deleting have their sync id's stored in `this._foldersToDelete`, - // and non-folders we plan on deleting have their sync id's stored in - // `this._atomsToDelete`. - // - The exception to this is the moves that occur to fix the order of bookmark - // children, which are performed after we process deletions. - // - Non-folders are deleted before folder deletions, so that when we process - // folder deletions we know the correct state. - // - Remote deletions always win for folders, but do not result in recursive - // deletion of children. This is a hack because we're not able to distinguish - // between value changes and structural changes to folders, and we don't even - // have the old server record to compare to. See `BookmarkEngine`'s - // `_shouldReviveRemotelyDeletedRecord` method. - // - When a folder is deleted, its remaining children are moved in order to - // their closest living ancestor. If this is interrupted (unlikely, but - // possible given that we don't perform this operation in a transaction), - // we revive the folder. - // - Remote deletions can lose for non-folders, but only until we handle - // bookmark restores correctly (removing stale state from the server -- this - // is to say, if bug 1230011 is fixed, we should never revive bookmarks). - - deletePending: Task.async(function* deletePending() { - yield this._deletePendingAtoms(); - let guidsToUpdate = yield this._deletePendingFolders(); - this.clearPendingDeletions(); - return guidsToUpdate; - }), - - clearPendingDeletions() { - this._foldersToDelete.clear(); - this._atomsToDelete.clear(); - }, - - _deleteAtom: Task.async(function* _deleteAtom(syncID) { - try { - let info = yield PlacesSyncUtils.bookmarks.remove(syncID, { - preventRemovalOfNonEmptyFolders: true - }); - this._log.trace(`Removed item ${syncID} with type ${info.type}`); - } catch (ex) { - // Likely already removed. - this._log.trace(`Error removing ${syncID}`, ex); + update: function BStore_update(record) { + let itemId = this.idForGUID(record.id); + + if (itemId <= 0) { + this._log.debug("Skipping update for unknown item: " + record.id); + return; } - }), - - _deletePendingAtoms() { - return Promise.all( - [...this._atomsToDelete.values()] - .map(syncID => this._deleteAtom(syncID))); - }, - - // Returns an array of sync ids that need updates. - _deletePendingFolders: Task.async(function* _deletePendingFolders() { - // To avoid data loss, we don't want to just delete the folder outright, - // so we buffer folder deletions and process them at the end (now). - // - // At this point, any member in the folder that remains is either a folder - // pending deletion (which we'll get to in this function), or an item that - // should not be deleted. To avoid deleting these items, we first move them - // to the parent of the folder we're about to delete. - let needUpdate = new Set(); - for (let syncId of this._foldersToDelete) { - let childSyncIds = yield PlacesSyncUtils.bookmarks.fetchChildSyncIds(syncId); - if (!childSyncIds.length) { - // No children -- just delete the folder. - yield this._deleteAtom(syncId) - continue; - } - // We could avoid some redundant work here by finding the nearest - // grandparent who isn't present in `this._toDelete`... - let grandparentSyncId = this.GUIDForId( - PlacesUtils.bookmarks.getFolderIdForItem( - this.idForGUID(PlacesSyncUtils.bookmarks.syncIdToGuid(syncId)))); + // Two items are the same type if they have the same ItemType in Places, + // and also share some key characteristics (e.g., both being livemarks). + // We figure this out by examining the item to find the equivalent granular + // (string) type. + // If they're not the same type, we can't just update attributes. Delete + // then recreate the record instead. + let localItemType = this._recordType(itemId); + let remoteRecordType = record.type; + this._log.trace("Local type: " + localItemType + ". " + + "Remote type: " + remoteRecordType + "."); + + if (localItemType != remoteRecordType) { + this._log.debug("Local record and remote record differ in type. " + + "Deleting and recreating."); + this.removeById(itemId, record.id); + this.create(record); + return; + } - this._log.trace(`Moving ${childSyncIds.length} children of "${syncId}" to ` + - `grandparent "${grandparentSyncId}" before deletion.`); + this._log.trace("Updating " + record.id + " (" + itemId + ")"); - // Move children out of the parent and into the grandparent - yield Promise.all(childSyncIds.map(child => PlacesSyncUtils.bookmarks.update({ - syncId: child, - parentSyncId: grandparentSyncId - }))); + // Move the bookmark to a new parent or new position if necessary + if (record._parent > 0 && + PlacesUtils.bookmarks.getFolderIdForItem(itemId) != record._parent) { + this._reparentItem(itemId, record._parent); + } - // Delete the (now empty) parent - try { - yield PlacesSyncUtils.bookmarks.remove(syncId, { - preventRemovalOfNonEmptyFolders: true - }); - } catch (e) { - // We failed, probably because someone added something to this folder - // between when we got the children and now (or the database is corrupt, - // or something else happened...) This is unlikely, but possible. To - // avoid corruption in this case, we need to reupload the record to the - // server. - // - // (Ideally this whole operation would be done in a transaction, and this - // wouldn't be possible). - needUpdate.add(syncId); + for (let [key, val] in Iterator(record.cleartext)) { + switch (key) { + case "title": + PlacesUtils.bookmarks.setItemTitle(itemId, val); + break; + case "bmkUri": + PlacesUtils.bookmarks.changeBookmarkURI(itemId, Utils.makeURI(val)); + break; + case "tags": + if (Array.isArray(val)) { + if (this.isTaggable(remoteRecordType)) { + this._tagID(itemId, val); + } else { + this._log.debug("Remote record type is invalid for tags: " + remoteRecordType); + } + } + break; + case "keyword": + PlacesUtils.bookmarks.setKeywordForBookmark(itemId, val); + break; + case "description": + if (val) { + PlacesUtils.annotations.setItemAnnotation( + itemId, DESCRIPTION_ANNO, val, 0, + PlacesUtils.annotations.EXPIRE_NEVER); + } else { + PlacesUtils.annotations.removeItemAnnotation(itemId, DESCRIPTION_ANNO); + } + break; + case "loadInSidebar": + if (val) { + PlacesUtils.annotations.setItemAnnotation( + itemId, SIDEBAR_ANNO, true, 0, + PlacesUtils.annotations.EXPIRE_NEVER); + } else { + PlacesUtils.annotations.removeItemAnnotation(itemId, SIDEBAR_ANNO); + } + break; + case "queryId": + PlacesUtils.annotations.setItemAnnotation( + itemId, SMART_BOOKMARKS_ANNO, val, 0, + PlacesUtils.annotations.EXPIRE_NEVER); + break; } + } + }, - // Add children (for parentid) and grandparent (for children list) to set - // of records needing an update, *unless* they're marked for deletion. - if (!this._foldersToDelete.has(grandparentSyncId)) { - needUpdate.add(grandparentSyncId); - } - for (let childSyncId of childSyncIds) { - if (!this._foldersToDelete.has(childSyncId)) { - needUpdate.add(childSyncId); + _orderChildren: function _orderChildren() { + for (let [guid, children] in Iterator(this._childrenToOrder)) { + // Reorder children according to the GUID list. Gracefully deal + // with missing items, e.g. locally deleted. + let delta = 0; + let parent = null; + for (let idx = 0; idx < children.length; idx++) { + let itemid = this.idForGUID(children[idx]); + if (itemid == -1) { + delta += 1; + this._log.trace("Could not locate record " + children[idx]); + continue; + } + try { + // This code path could be optimized by caching the parent earlier. + // Doing so should take in count any edge case due to reparenting + // or parent invalidations though. + if (!parent) { + parent = PlacesUtils.bookmarks.getFolderIdForItem(itemid); + } + PlacesUtils.bookmarks.moveItem(itemid, parent, idx - delta); + } catch (ex) { + this._log.debug("Could not move item " + children[idx] + ": " + ex); } } } - return [...needUpdate]; - }), + }, changeItemID: function BStore_changeItemID(oldID, newID) { this._log.debug("Changing GUID " + oldID + " to " + newID); - Async.promiseSpinningly(PlacesSyncUtils.bookmarks.changeGuid(oldID, newID)); + // Make sure there's an item to change GUIDs + let itemId = this.idForGUID(oldID); + if (itemId <= 0) + return; + + this._setGUID(itemId, newID); + }, + + _getNode: function BStore__getNode(folder) { + let query = PlacesUtils.history.getNewQuery(); + query.setFolders([folder], 1); + return PlacesUtils.history.executeQuery( + query, PlacesUtils.history.getNewQueryOptions()).root; + }, + + _getTags: function BStore__getTags(uri) { + try { + if (typeof(uri) == "string") + uri = Utils.makeURI(uri); + } catch(e) { + this._log.warn("Could not parse URI \"" + uri + "\": " + e); + } + return PlacesUtils.tagging.getTagsForURI(uri, {}); + }, + + _getDescription: function BStore__getDescription(id) { + try { + return PlacesUtils.annotations.getItemAnnotation(id, DESCRIPTION_ANNO); + } catch (e) { + return null; + } + }, + + _isLoadInSidebar: function BStore__isLoadInSidebar(id) { + return PlacesUtils.annotations.itemHasAnnotation(id, SIDEBAR_ANNO); + }, + + get _childGUIDsStm() { + return this._getStmt( + "SELECT id AS item_id, guid " + + "FROM moz_bookmarks " + + "WHERE parent = :parent " + + "ORDER BY position"); + }, + _childGUIDsCols: ["item_id", "guid"], + + _getChildGUIDsForId: function _getChildGUIDsForId(itemid) { + let stmt = this._childGUIDsStm; + stmt.params.parent = itemid; + let rows = Async.querySpinningly(stmt, this._childGUIDsCols); + return rows.map(function (row) { + if (row.guid) { + return row.guid; + } + // A GUID hasn't been assigned to this item yet, do this now. + return this.GUIDForId(row.item_id); + }, this); }, // Create a record starting from the weave id (places guid) createRecord: function createRecord(id, collection) { - let item = Async.promiseSpinningly(PlacesSyncUtils.bookmarks.fetch(id)); - if (!item) { // deleted item - let record = new PlacesItem(collection, id); + let placeId = this.idForGUID(id); + let record; + if (placeId <= 0) { // deleted item + record = new PlacesItem(collection, id); record.deleted = true; return record; } - let recordObj = getTypeObject(item.kind); - if (!recordObj) { - this._log.warn("Unknown item type, cannot serialize: " + item.kind); - recordObj = PlacesItem; + let parent = PlacesUtils.bookmarks.getFolderIdForItem(placeId); + switch (PlacesUtils.bookmarks.getItemType(placeId)) { + case PlacesUtils.bookmarks.TYPE_BOOKMARK: + let bmkUri = PlacesUtils.bookmarks.getBookmarkURI(placeId).spec; + if (bmkUri.indexOf("place:") == 0) { + record = new BookmarkQuery(collection, id); + + // Get the actual tag name instead of the local itemId + let folder = bmkUri.match(/[:&]folder=(\d+)/); + try { + // There might not be the tag yet when creating on a new client + if (folder != null) { + folder = folder[1]; + record.folderName = PlacesUtils.bookmarks.getItemTitle(folder); + this._log.trace("query id: " + folder + " = " + record.folderName); + } + } + catch(ex) {} + + // Persist the Smart Bookmark anno, if found. + try { + let anno = PlacesUtils.annotations.getItemAnnotation(placeId, SMART_BOOKMARKS_ANNO); + if (anno != null) { + this._log.trace("query anno: " + SMART_BOOKMARKS_ANNO + + " = " + anno); + record.queryId = anno; + } + } + catch(ex) {} + } + else { + record = new Bookmark(collection, id); + } + record.title = PlacesUtils.bookmarks.getItemTitle(placeId); + + record.parentName = PlacesUtils.bookmarks.getItemTitle(parent); + record.bmkUri = bmkUri; + record.tags = this._getTags(record.bmkUri); + record.keyword = PlacesUtils.bookmarks.getKeywordForBookmark(placeId); + record.description = this._getDescription(placeId); + record.loadInSidebar = this._isLoadInSidebar(placeId); + break; + + case PlacesUtils.bookmarks.TYPE_FOLDER: + if (PlacesUtils.annotations + .itemHasAnnotation(placeId, PlacesUtils.LMANNO_FEEDURI)) { + record = new Livemark(collection, id); + let as = PlacesUtils.annotations; + record.feedUri = as.getItemAnnotation(placeId, PlacesUtils.LMANNO_FEEDURI); + try { + record.siteUri = as.getItemAnnotation(placeId, PlacesUtils.LMANNO_SITEURI); + } catch (ex) {} + } else { + record = new BookmarkFolder(collection, id); + } + + if (parent > 0) + record.parentName = PlacesUtils.bookmarks.getItemTitle(parent); + record.title = PlacesUtils.bookmarks.getItemTitle(placeId); + record.description = this._getDescription(placeId); + record.children = this._getChildGUIDsForId(placeId); + break; + + case PlacesUtils.bookmarks.TYPE_SEPARATOR: + record = new BookmarkSeparator(collection, id); + if (parent > 0) + record.parentName = PlacesUtils.bookmarks.getItemTitle(parent); + // Create a positioning identifier for the separator, used by _mapDupe + record.pos = PlacesUtils.bookmarks.getItemIndex(placeId); + break; + + default: + record = new PlacesItem(collection, id); + this._log.warn("Unknown item type, cannot serialize: " + + PlacesUtils.bookmarks.getItemType(placeId)); } - let record = new recordObj(collection, id); - record.fromSyncBookmark(item); + record.parentid = this.GUIDForId(parent); record.sortindex = this._calculateIndex(record); return record; @@ -997,22 +1116,84 @@ BookmarksStore.prototype = { return this._getStmt( "SELECT frecency " + "FROM moz_places " + - "WHERE url_hash = hash(:url) AND url = :url " + + "WHERE url = :url " + "LIMIT 1"); }, _frecencyCols: ["frecency"], + get _setGUIDStm() { + return this._getStmt( + "UPDATE moz_bookmarks " + + "SET guid = :guid " + + "WHERE id = :item_id"); + }, + + // Some helper functions to handle GUIDs + _setGUID: function _setGUID(id, guid) { + if (!guid) + guid = Utils.makeGUID(); + + let stmt = this._setGUIDStm; + stmt.params.guid = guid; + stmt.params.item_id = id; + Async.querySpinningly(stmt); + return guid; + }, + + get _guidForIdStm() { + return this._getStmt( + "SELECT guid " + + "FROM moz_bookmarks " + + "WHERE id = :item_id"); + }, + _guidForIdCols: ["guid"], + GUIDForId: function GUIDForId(id) { - let guid = Async.promiseSpinningly(PlacesUtils.promiseItemGuid(id)); - return PlacesSyncUtils.bookmarks.guidToSyncId(guid); + let special = kSpecialIds.specialGUIDForId(id); + if (special) + return special; + + let stmt = this._guidForIdStm; + stmt.params.item_id = id; + + // Use the existing GUID if it exists + let result = Async.querySpinningly(stmt, this._guidForIdCols)[0]; + if (result && result.guid) + return result.guid; + + // Give the uri a GUID if it doesn't have one + return this._setGUID(id); }, - idForGUID: function idForGUID(guid) { - // guid might be a String object rather than a string. - guid = PlacesSyncUtils.bookmarks.syncIdToGuid(guid.toString()); + get _idForGUIDStm() { + return this._getStmt( + "SELECT id AS item_id " + + "FROM moz_bookmarks " + + "WHERE guid = :guid"); + }, + _idForGUIDCols: ["item_id"], + + // noCreate is provided as an optional argument to prevent the creation of + // non-existent special records, such as "mobile". + idForGUID: function idForGUID(guid, noCreate) { + if (kSpecialIds.isSpecialGUID(guid)) + return kSpecialIds.specialIdForGUID(guid, !noCreate); - return Async.promiseSpinningly(PlacesUtils.promiseItemId(guid).catch( - ex => -1)); + let stmt = this._idForGUIDStm; + // guid might be a String object rather than a string. + stmt.params.guid = guid.toString(); + + let results = Async.querySpinningly(stmt, this._idForGUIDCols); + this._log.trace("Number of rows matching GUID " + guid + ": " + + results.length); + + // Here's the one we care about: the first. + let result = results[0]; + + if (!result) + return -1; + + return result.item_id; }, _calculateIndex: function _calculateIndex(record) { @@ -1037,48 +1218,106 @@ BookmarksStore.prototype = { return index; }, - getAllIDs: function BStore_getAllIDs() { - let items = {}; - - let query = ` - WITH RECURSIVE - changeRootContents(id) AS ( - VALUES ${getChangeRootIds().map(id => `(${id})`).join(", ")} - UNION ALL - SELECT b.id - FROM moz_bookmarks b - JOIN changeRootContents c ON b.parent = c.id - ) - SELECT guid - FROM changeRootContents - JOIN moz_bookmarks USING (id) - `; - - let statement = this._getStmt(query); - let results = Async.querySpinningly(statement, ["guid"]); - for (let { guid } of results) { - let syncID = PlacesSyncUtils.bookmarks.guidToSyncId(guid); - items[syncID] = { modified: 0, deleted: false }; + _getChildren: function BStore_getChildren(guid, items) { + let node = guid; // the recursion case + if (typeof(node) == "string") { // callers will give us the guid as the first arg + let nodeID = this.idForGUID(guid, true); + if (!nodeID) { + this._log.debug("No node for GUID " + guid + "; returning no children."); + return items; + } + node = this._getNode(nodeID); + } + + if (node.type == node.RESULT_TYPE_FOLDER) { + node.QueryInterface(Ci.nsINavHistoryQueryResultNode); + node.containerOpen = true; + try { + // Remember all the children GUIDs and recursively get more + for (let i = 0; i < node.childCount; i++) { + let child = node.getChild(i); + items[this.GUIDForId(child.itemId)] = true; + this._getChildren(child, items); + } + } + finally { + node.containerOpen = false; + } } return items; }, + /** + * Associates the URI of the item with the provided ID with the + * provided array of tags. + * If the provided ID does not identify an item with a URI, + * returns immediately. + */ + _tagID: function _tagID(itemID, tags) { + if (!itemID || !tags) { + return; + } + + try { + let u = PlacesUtils.bookmarks.getBookmarkURI(itemID); + this._tagURI(u, tags); + } catch (e) { + this._log.warn("Got exception fetching URI for " + itemID + ": not tagging. ", e); + + // I guess it doesn't have a URI. Don't try to tag it. + return; + } + }, + + /** + * Associate the provided URI with the provided array of tags. + * If the provided URI is falsy, returns immediately. + */ + _tagURI: function _tagURI(bookmarkURI, tags) { + if (!bookmarkURI || !tags) { + return; + } + + // Filter out any null/undefined/empty tags. + tags = tags.filter(t => t); + + // Temporarily tag a dummy URI to preserve tag ids when untagging. + let dummyURI = Utils.makeURI("about:weave#BStore_tagURI"); + PlacesUtils.tagging.tagURI(dummyURI, tags); + PlacesUtils.tagging.untagURI(bookmarkURI, null); + PlacesUtils.tagging.tagURI(bookmarkURI, tags); + PlacesUtils.tagging.untagURI(dummyURI, null); + }, + + getAllIDs: function BStore_getAllIDs() { + let items = {"menu": true, + "toolbar": true}; + for each (let guid in kSpecialIds.guids) { + if (guid != "places" && guid != "tags") + this._getChildren(guid, items); + } + return items; + }, + wipe: function BStore_wipe() { - this.clearPendingDeletions(); - Async.promiseSpinningly(Task.spawn(function* () { + let cb = Async.makeSpinningCallback(); + Task.spawn(function() { // Save a backup before clearing out all bookmarks. yield PlacesBackups.create(null, true); - yield PlacesUtils.bookmarks.eraseEverything({ - source: SOURCE_SYNC, - }); - })); + for each (let guid in kSpecialIds.guids) + if (guid != "places") { + let id = kSpecialIds.specialIdForGUID(guid); + if (id) + PlacesUtils.bookmarks.removeFolderChildren(id); + } + cb(); + }); + cb.wait(); } }; function BookmarksTracker(name, engine) { - this._batchDepth = 0; - this._batchSawScoreIncrement = false; Tracker.call(this, name, engine); Svc.Obs.add("places-shutdown", this); @@ -1086,16 +1325,6 @@ function BookmarksTracker(name, engine) { BookmarksTracker.prototype = { __proto__: Tracker.prototype, - //`_ignore` checks the change source for each observer notification, so we - // don't want to let the engine ignore all changes during a sync. - get ignoreAll() { - return false; - }, - - // Define an empty setter so that the engine doesn't throw a `TypeError` - // setting a read-only property. - set ignoreAll(value) {}, - startTracking: function() { PlacesUtils.bookmarks.addObserver(this, true); Svc.Obs.add("bookmarks-restore-begin", this); @@ -1116,9 +1345,11 @@ BookmarksTracker.prototype = { switch (topic) { case "bookmarks-restore-begin": this._log.debug("Ignoring changes from importing bookmarks."); + this.ignoreAll = true; break; case "bookmarks-restore-success": this._log.debug("Tracking all items on successful import."); + this.ignoreAll = false; this._log.debug("Restore succeeded: wiping server and other clients."); this.engine.service.resetClient([this.name]); @@ -1127,6 +1358,7 @@ BookmarksTracker.prototype = { break; case "bookmarks-restore-failed": this._log.debug("Tracking all items on failed import."); + this.ignoreAll = false; break; } }, @@ -1137,68 +1369,73 @@ BookmarksTracker.prototype = { Ci.nsISupportsWeakReference ]), - addChangedID(id, change) { - if (!id) { - this._log.warn("Attempted to add undefined ID to tracker"); - return false; - } - if (this._ignored.includes(id)) { - return false; - } - let shouldSaveChange = false; - let currentChange = this.changedIDs[id]; - if (currentChange) { - if (typeof currentChange == "number") { - // Allow raw timestamps for backward-compatibility with persisted - // changed IDs. The new format uses tuples to track deleted items. - shouldSaveChange = currentChange < change.modified; - } else { - shouldSaveChange = currentChange.modified < change.modified || - currentChange.deleted != change.deleted; - } - } else { - shouldSaveChange = true; - } - if (shouldSaveChange) { - this._saveChangedID(id, change); - } - return true; - }, - /** * Add a bookmark GUID to be uploaded and bump up the sync score. * - * @param itemId - * The Places item ID of the bookmark to upload. - * @param guid - * The Places GUID of the bookmark to upload. - * @param isTombstone - * Whether we're uploading a tombstone for a removed bookmark. + * @param itemGuid + * GUID of the bookmark to upload. */ - _add: function BMT__add(itemId, guid, isTombstone = false) { - let syncID = PlacesSyncUtils.bookmarks.guidToSyncId(guid); - let info = { modified: Date.now() / 1000, deleted: isTombstone }; - if (this.addChangedID(syncID, info)) { + _add: function BMT__add(itemId, guid) { + guid = kSpecialIds.specialGUIDForId(itemId) || guid; + if (this.addChangedID(guid)) this._upScore(); - } }, - /* Every add/remove/change will trigger a sync for MULTI_DEVICE (except in - a batch operation, where we do it at the end of the batch) */ + /* Every add/remove/change will trigger a sync for MULTI_DEVICE. */ _upScore: function BMT__upScore() { - if (this._batchDepth == 0) { - this.score += SCORE_INCREMENT_XLARGE; - } else { - this._batchSawScoreIncrement = true; + this.score += SCORE_INCREMENT_XLARGE; + }, + + /** + * Determine if a change should be ignored. + * + * @param itemId + * Item under consideration to ignore + * @param folder (optional) + * Folder of the item being changed + */ + _ignore: function BMT__ignore(itemId, folder, guid) { + // Ignore unconditionally if the engine tells us to. + if (this.ignoreAll) + return true; + + // Get the folder id if we weren't given one. + if (folder == null) { + try { + folder = PlacesUtils.bookmarks.getFolderIdForItem(itemId); + } catch (ex) { + this._log.debug("getFolderIdForItem(" + itemId + + ") threw; calling _ensureMobileQuery."); + // I'm guessing that gFIFI can throw, and perhaps that's why + // _ensureMobileQuery is here at all. Try not to call it. + this._ensureMobileQuery(); + folder = PlacesUtils.bookmarks.getFolderIdForItem(itemId); + } + } + + // Ignore changes to tags (folders under the tags folder). + let tags = kSpecialIds.tags; + if (folder == tags) + return true; + + // Ignore tag items (the actual instance of a tag for a bookmark). + if (PlacesUtils.bookmarks.getFolderIdForItem(folder) == tags) + return true; + + // Make sure to remove items that have the exclude annotation. + if (PlacesUtils.annotations.itemHasAnnotation(itemId, EXCLUDEBACKUP_ANNO)) { + this.removeChangedID(guid); + return true; } + + return false; }, onItemAdded: function BMT_onItemAdded(itemId, folder, index, itemType, uri, title, dateAdded, - guid, parentGuid, source) { - if (IGNORED_SOURCES.includes(source)) { + guid, parentGuid) { + if (this._ignore(itemId, folder, guid)) return; - } this._log.trace("onItemAdded: " + itemId); this._add(itemId, guid); @@ -1206,51 +1443,13 @@ BookmarksTracker.prototype = { }, onItemRemoved: function (itemId, parentId, index, type, uri, - guid, parentGuid, source) { - if (IGNORED_SOURCES.includes(source)) { - return; - } - - // Ignore changes to tags (folders under the tags folder). - if (parentId == PlacesUtils.tagsFolderId) { - return; - } - - let grandParentId = -1; - try { - grandParentId = PlacesUtils.bookmarks.getFolderIdForItem(parentId); - } catch (ex) { - // `getFolderIdForItem` can throw if the item no longer exists, such as - // when we've removed a subtree using `removeFolderChildren`. - return; - } - - // Ignore tag items (the actual instance of a tag for a bookmark). - if (grandParentId == PlacesUtils.tagsFolderId) { + guid, parentGuid) { + if (this._ignore(itemId, parentId, guid)) { return; } - /** - * The above checks are incomplete: we can still write tombstones for - * items that we don't track, and upload extraneous roots. - * - * Consider the left pane root: it's a child of the Places root, and has - * children and grandchildren. `PlacesUIUtils` can create, delete, and - * recreate it as needed. We can't determine ancestors when the root or its - * children are deleted, because they've already been removed from the - * database when `onItemRemoved` is called. Likewise, we can't check their - * "exclude from backup" annos, because they've *also* been removed. - * - * So, we end up writing tombstones for the left pane queries and left - * pane root. For good measure, we'll also upload the Places root, because - * it's the parent of the left pane root. - * - * As a workaround, we can track the parent GUID and reconstruct the item's - * ancestry at sync time. This is complicated, and the previous behavior was - * already wrong, so we'll wait for bug 1258127 to fix this generally. - */ this._log.trace("onItemRemoved: " + itemId); - this._add(itemId, guid, /* isTombstone */ true); + this._add(itemId, guid); this._add(parentId, parentGuid); }, @@ -1265,40 +1464,32 @@ BookmarksTracker.prototype = { if (all.length == 0) return; + // Disable handling of notifications while changing the mobile query + this.ignoreAll = true; + let mobile = find(MOBILE_ANNO); - let queryURI = Utils.makeURI("place:folder=" + PlacesUtils.mobileFolderId); - let title = PlacesBundle.GetStringFromName("MobileBookmarksFolderTitle"); + let queryURI = Utils.makeURI("place:folder=" + kSpecialIds.mobile); + let title = Str.sync.get("mobile.label"); // Don't add OR remove the mobile bookmarks if there's nothing. - if (PlacesUtils.bookmarks.getIdForItemAt(PlacesUtils.mobileFolderId, 0) == -1) { + if (PlacesUtils.bookmarks.getIdForItemAt(kSpecialIds.mobile, 0) == -1) { if (mobile.length != 0) - PlacesUtils.bookmarks.removeItem(mobile[0], SOURCE_SYNC); + PlacesUtils.bookmarks.removeItem(mobile[0]); } // Add the mobile bookmarks query if it doesn't exist else if (mobile.length == 0) { - let query = PlacesUtils.bookmarks.insertBookmark(all[0], queryURI, -1, title, /* guid */ null, SOURCE_SYNC); + let query = PlacesUtils.bookmarks.insertBookmark(all[0], queryURI, -1, title); PlacesUtils.annotations.setItemAnnotation(query, ORGANIZERQUERY_ANNO, MOBILE_ANNO, 0, - PlacesUtils.annotations.EXPIRE_NEVER, SOURCE_SYNC); - PlacesUtils.annotations.setItemAnnotation(query, PlacesUtils.EXCLUDE_FROM_BACKUP_ANNO, 1, 0, - PlacesUtils.annotations.EXPIRE_NEVER, SOURCE_SYNC); + PlacesUtils.annotations.EXPIRE_NEVER); + PlacesUtils.annotations.setItemAnnotation(query, EXCLUDEBACKUP_ANNO, 1, 0, + PlacesUtils.annotations.EXPIRE_NEVER); } - // Make sure the existing query URL and title are correct - else { - if (!PlacesUtils.bookmarks.getBookmarkURI(mobile[0]).equals(queryURI)) { - PlacesUtils.bookmarks.changeBookmarkURI(mobile[0], queryURI, - SOURCE_SYNC); - } - let queryTitle = PlacesUtils.bookmarks.getItemTitle(mobile[0]); - if (queryTitle != title) { - PlacesUtils.bookmarks.setItemTitle(mobile[0], title, SOURCE_SYNC); - } - let rootTitle = - PlacesUtils.bookmarks.getItemTitle(PlacesUtils.mobileFolderId); - if (rootTitle != title) { - PlacesUtils.bookmarks.setItemTitle(PlacesUtils.mobileFolderId, title, - SOURCE_SYNC); - } + // Make sure the existing title is correct + else if (PlacesUtils.bookmarks.getItemTitle(mobile[0]) != title) { + PlacesUtils.bookmarks.setItemTitle(mobile[0], title); } + + this.ignoreAll = false; }, // This method is oddly structured, but the idea is to return as quickly as @@ -1306,11 +1497,10 @@ BookmarksTracker.prototype = { // *each change*. onItemChanged: function BMT_onItemChanged(itemId, property, isAnno, value, lastModified, itemType, parentId, - guid, parentGuid, oldValue, - source) { - if (IGNORED_SOURCES.includes(source)) { + guid, parentGuid) { + // Quicker checks first. + if (this.ignoreAll) return; - } if (isAnno && (ANNOS_TO_TRACK.indexOf(property) == -1)) // Ignore annotations except for the ones that we sync. @@ -1320,6 +1510,9 @@ BookmarksTracker.prototype = { if (property == "favicon") return; + if (this._ignore(itemId, parentId, guid)) + return; + this._log.trace("onItemChanged: " + itemId + (", " + property + (isAnno? " (anno)" : "")) + (value ? (" = \"" + value + "\"") : "")); @@ -1328,11 +1521,9 @@ BookmarksTracker.prototype = { onItemMoved: function BMT_onItemMoved(itemId, oldParent, oldIndex, newParent, newIndex, itemType, - guid, oldParentGuid, newParentGuid, - source) { - if (IGNORED_SOURCES.includes(source)) { + guid, oldParentGuid, newParentGuid) { + if (this._ignore(itemId, newParent, guid)) return; - } this._log.trace("onItemMoved: " + itemId); this._add(oldParent, oldParentGuid); @@ -1342,37 +1533,10 @@ BookmarksTracker.prototype = { } // Remove any position annotations now that the user moved the item - PlacesUtils.annotations.removeItemAnnotation(itemId, - PlacesSyncUtils.bookmarks.SYNC_PARENT_ANNO, SOURCE_SYNC); + PlacesUtils.annotations.removeItemAnnotation(itemId, PARENT_ANNO); }, - onBeginUpdateBatch: function () { - ++this._batchDepth; - }, - onEndUpdateBatch: function () { - if (--this._batchDepth === 0 && this._batchSawScoreIncrement) { - this.score += SCORE_INCREMENT_XLARGE; - this._batchSawScoreIncrement = false; - } - }, + onBeginUpdateBatch: function () {}, + onEndUpdateBatch: function () {}, onItemVisited: function () {} }; - -// Returns an array of root IDs to recursively query for synced bookmarks. -// Items in other roots, including tags and organizer queries, will be -// ignored. -function getChangeRootIds() { - return [ - PlacesUtils.bookmarksMenuFolderId, - PlacesUtils.toolbarFolderId, - PlacesUtils.unfiledBookmarksFolderId, - PlacesUtils.mobileFolderId, - ]; -} - -class BookmarksChangeset extends Changeset { - getModifiedTimestamp(id) { - let change = this.changes[id]; - return change ? change.modified : Number.NaN; - } -} diff --git a/services/sync/modules/engines/clients.js b/services/sync/modules/engines/clients.js index 3dd679570..6c8e37a7b 100644 --- a/services/sync/modules/engines/clients.js +++ b/services/sync/modules/engines/clients.js @@ -2,24 +2,6 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -/** - * How does the clients engine work? - * - * - We use 2 files - commands.json and commands-syncing.json. - * - * - At sync upload time, we attempt a rename of commands.json to - * commands-syncing.json, and ignore errors (helps for crash during sync!). - * - We load commands-syncing.json and stash the contents in - * _currentlySyncingCommands which lives for the duration of the upload process. - * - We use _currentlySyncingCommands to build the outgoing records - * - Immediately after successful upload, we delete commands-syncing.json from - * disk (and clear _currentlySyncingCommands). We reconcile our local records - * with what we just wrote in the server, and add failed IDs commands - * back in commands.json - * - Any time we need to "save" a command for future syncs, we load - * commands.json, update it, and write it back out. - */ - this.EXPORTED_SYMBOLS = [ "ClientEngine", "ClientsRec" @@ -27,32 +9,17 @@ this.EXPORTED_SYMBOLS = [ var {classes: Cc, interfaces: Ci, utils: Cu} = Components; -Cu.import("resource://services-common/async.js"); Cu.import("resource://services-common/stringbundle.js"); Cu.import("resource://services-sync/constants.js"); Cu.import("resource://services-sync/engines.js"); Cu.import("resource://services-sync/record.js"); -Cu.import("resource://services-sync/resource.js"); Cu.import("resource://services-sync/util.js"); -Cu.import("resource://gre/modules/Services.jsm"); - -XPCOMUtils.defineLazyModuleGetter(this, "fxAccounts", - "resource://gre/modules/FxAccounts.jsm"); const CLIENTS_TTL = 1814400; // 21 days const CLIENTS_TTL_REFRESH = 604800; // 7 days -const STALE_CLIENT_REMOTE_AGE = 604800; // 7 days const SUPPORTED_PROTOCOL_VERSIONS = ["1.1", "1.5"]; -function hasDupeCommand(commands, action) { - if (!commands) { - return false; - } - return commands.some(other => other.command == action.command && - Utils.deepEquals(other.args, action.args)); -} - this.ClientsRec = function ClientsRec(collection, id) { CryptoWrapper.call(this, collection, id); } @@ -66,27 +33,23 @@ Utils.deferGetSet(ClientsRec, "cleartext", ["name", "type", "commands", "version", "protocols", - "formfactor", "os", "appPackage", "application", "device", - "fxaDeviceId"]); + "formfactor", "os", "appPackage", "application", "device"]); this.ClientEngine = function ClientEngine(service) { SyncEngine.call(this, "Clients", service); - // Reset the last sync timestamp on every startup so that we fetch all clients - this.resetLastSync(); + // Reset the client on every startup so that we fetch recent clients + this._resetClient(); } ClientEngine.prototype = { __proto__: SyncEngine.prototype, _storeObj: ClientStore, _recordObj: ClientsRec, _trackerObj: ClientsTracker, - allowSkippedRecord: false, // Always sync client data as it controls other sync behavior - get enabled() { - return true; - }, + get enabled() true, get lastRecordUpload() { return Svc.Prefs.get(this.name + ".lastRecordUpload", 0); @@ -95,20 +58,10 @@ ClientEngine.prototype = { Svc.Prefs.set(this.name + ".lastRecordUpload", Math.floor(value)); }, - get remoteClients() { - // return all non-stale clients for external consumption. - return Object.values(this._store._remoteClients).filter(v => !v.stale); - }, - - remoteClientExists(id) { - let client = this._store._remoteClients[id]; - return !!(client && !client.stale); - }, - // Aggregate some stats on the composition of clients on this account get stats() { let stats = { - hasMobile: this.localType == DEVICE_TYPE_MOBILE, + hasMobile: this.localType == "mobile", names: [this.localName], numClients: 1, }; @@ -156,9 +109,7 @@ ClientEngine.prototype = { let localID = Svc.Prefs.get("client.GUID", ""); return localID == "" ? this.localID = Utils.makeGUID() : localID; }, - set localID(value) { - Svc.Prefs.set("client.GUID", value); - }, + set localID(value) Svc.Prefs.set("client.GUID", value), get brandName() { let brand = new StringBundle("chrome://branding/locale/brand.properties"); @@ -166,97 +117,23 @@ ClientEngine.prototype = { }, get localName() { - let name = Utils.getDeviceName(); - // If `getDeviceName` returns the default name, set the pref. FxA registers - // the device before syncing, so we don't need to update the registration - // in this case. - Svc.Prefs.set("client.name", name); - return name; - }, - set localName(value) { - Svc.Prefs.set("client.name", value); - // Update the registration in the background. - fxAccounts.updateDeviceRegistration().catch(error => { - this._log.warn("failed to update fxa device registration", error); - }); - }, + let localName = Svc.Prefs.get("client.name", ""); + if (localName != "") + return localName; - get localType() { - return Utils.getDeviceType(); - }, - set localType(value) { - Svc.Prefs.set("client.type", value); + return this.localName = Utils.getDefaultDeviceName(); }, + set localName(value) Svc.Prefs.set("client.name", value), - getClientName(id) { - if (id == this.localID) { - return this.localName; - } - let client = this._store._remoteClients[id]; - return client ? client.name : ""; - }, - - getClientFxaDeviceId(id) { - if (this._store._remoteClients[id]) { - return this._store._remoteClients[id].fxaDeviceId; - } - return null; - }, + get localType() Svc.Prefs.get("client.type", "desktop"), + set localType(value) Svc.Prefs.set("client.type", value), isMobile: function isMobile(id) { if (this._store._remoteClients[id]) - return this._store._remoteClients[id].type == DEVICE_TYPE_MOBILE; + return this._store._remoteClients[id].type == "mobile"; return false; }, - _readCommands() { - let cb = Async.makeSpinningCallback(); - Utils.jsonLoad("commands", this, commands => cb(null, commands)); - return cb.wait() || {}; - }, - - /** - * Low level function, do not use directly (use _addClientCommand instead). - */ - _saveCommands(commands) { - let cb = Async.makeSpinningCallback(); - Utils.jsonSave("commands", this, commands, error => { - if (error) { - this._log.error("Failed to save JSON outgoing commands", error); - } - cb(); - }); - cb.wait(); - }, - - _prepareCommandsForUpload() { - let cb = Async.makeSpinningCallback(); - Utils.jsonMove("commands", "commands-syncing", this).catch(() => {}) // Ignore errors - .then(() => { - Utils.jsonLoad("commands-syncing", this, commands => cb(null, commands)); - }); - return cb.wait() || {}; - }, - - _deleteUploadedCommands() { - delete this._currentlySyncingCommands; - Async.promiseSpinningly( - Utils.jsonRemove("commands-syncing", this).catch(err => { - this._log.error("Failed to delete syncing-commands file", err); - }) - ); - }, - - _addClientCommand(clientId, command) { - const allCommands = this._readCommands(); - const clientCommands = allCommands[clientId] || []; - if (hasDupeCommand(clientCommands, command)) { - return; - } - allCommands[clientId] = clientCommands.concat(command); - this._saveCommands(allCommands); - }, - _syncStartup: function _syncStartup() { // Reupload new client record periodically. if (Date.now() / 1000 - this.lastRecordUpload > CLIENTS_TTL_REFRESH) { @@ -266,157 +143,9 @@ ClientEngine.prototype = { SyncEngine.prototype._syncStartup.call(this); }, - _processIncoming() { - // Fetch all records from the server. - this.lastSync = 0; - this._incomingClients = {}; - try { - SyncEngine.prototype._processIncoming.call(this); - // Since clients are synced unconditionally, any records in the local store - // that don't exist on the server must be for disconnected clients. Remove - // them, so that we don't upload records with commands for clients that will - // never see them. We also do this to filter out stale clients from the - // tabs collection, since showing their list of tabs is confusing. - for (let id in this._store._remoteClients) { - if (!this._incomingClients[id]) { - this._log.info(`Removing local state for deleted client ${id}`); - this._removeRemoteClient(id); - } - } - // Bug 1264498: Mobile clients don't remove themselves from the clients - // collection when the user disconnects Sync, so we mark as stale clients - // with the same name that haven't synced in over a week. - // (Note we can't simply delete them, or we re-apply them next sync - see - // bug 1287687) - delete this._incomingClients[this.localID]; - let names = new Set([this.localName]); - for (let id in this._incomingClients) { - let record = this._store._remoteClients[id]; - if (!names.has(record.name)) { - names.add(record.name); - continue; - } - let remoteAge = AsyncResource.serverTime - this._incomingClients[id]; - if (remoteAge > STALE_CLIENT_REMOTE_AGE) { - this._log.info(`Hiding stale client ${id} with age ${remoteAge}`); - record.stale = true; - } - } - } finally { - this._incomingClients = null; - } - }, - - _uploadOutgoing() { - this._currentlySyncingCommands = this._prepareCommandsForUpload(); - const clientWithPendingCommands = Object.keys(this._currentlySyncingCommands); - for (let clientId of clientWithPendingCommands) { - if (this._store._remoteClients[clientId] || this.localID == clientId) { - this._modified.set(clientId, 0); - } - } - SyncEngine.prototype._uploadOutgoing.call(this); - }, - - _onRecordsWritten(succeeded, failed) { - // Reconcile the status of the local records with what we just wrote on the - // server - for (let id of succeeded) { - const commandChanges = this._currentlySyncingCommands[id]; - if (id == this.localID) { - if (this.localCommands) { - this.localCommands = this.localCommands.filter(command => !hasDupeCommand(commandChanges, command)); - } - } else { - const clientRecord = this._store._remoteClients[id]; - if (!commandChanges || !clientRecord) { - // should be impossible, else we wouldn't have been writing it. - this._log.warn("No command/No record changes for a client we uploaded"); - continue; - } - // fixup the client record, so our copy of _remoteClients matches what we uploaded. - clientRecord.commands = this._store.createRecord(id); - // we could do better and pass the reference to the record we just uploaded, - // but this will do for now - } - } - - // Re-add failed commands - for (let id of failed) { - const commandChanges = this._currentlySyncingCommands[id]; - if (!commandChanges) { - continue; - } - this._addClientCommand(id, commandChanges); - } - - this._deleteUploadedCommands(); - - // Notify other devices that their own client collection changed - const idsToNotify = succeeded.reduce((acc, id) => { - if (id == this.localID) { - return acc; - } - const fxaDeviceId = this.getClientFxaDeviceId(id); - return fxaDeviceId ? acc.concat(fxaDeviceId) : acc; - }, []); - if (idsToNotify.length > 0) { - this._notifyCollectionChanged(idsToNotify); - } - }, - - _notifyCollectionChanged(ids) { - const message = { - version: 1, - command: "sync:collection_changed", - data: { - collections: ["clients"] - } - }; - fxAccounts.notifyDevices(ids, message, NOTIFY_TAB_SENT_TTL_SECS); - }, - - _syncFinish() { - // Record histograms for our device types, and also write them to a pref - // so non-histogram telemetry (eg, UITelemetry) has easy access to them. - for (let [deviceType, count] of this.deviceTypes) { - let hid; - let prefName = this.name + ".devices."; - switch (deviceType) { - case "desktop": - hid = "WEAVE_DEVICE_COUNT_DESKTOP"; - prefName += "desktop"; - break; - case "mobile": - hid = "WEAVE_DEVICE_COUNT_MOBILE"; - prefName += "mobile"; - break; - default: - this._log.warn(`Unexpected deviceType "${deviceType}" recording device telemetry.`); - continue; - } - Services.telemetry.getHistogramById(hid).add(count); - Svc.Prefs.set(prefName, count); - } - SyncEngine.prototype._syncFinish.call(this); - }, - - _reconcile: function _reconcile(item) { - // Every incoming record is reconciled, so we use this to track the - // contents of the collection on the server. - this._incomingClients[item.id] = item.modified; - - if (!this._store.itemExists(item.id)) { - return true; - } - // Clients are synced unconditionally, so we'll always have new records. - // Unfortunately, this will cause the scheduler to use the immediate sync - // interval for the multi-device case, instead of the active interval. We - // work around this by updating the record during reconciliation, and - // returning false to indicate that the record doesn't need to be applied - // later. - this._store.update(item); - return false; + // Always process incoming items because they might have commands + _reconcile: function _reconcile() { + return true; }, // Treat reset the same as wiping for locally cached clients @@ -426,13 +155,7 @@ ClientEngine.prototype = { _wipeClient: function _wipeClient() { SyncEngine.prototype._resetClient.call(this); - delete this.localCommands; this._store.wipe(); - const logRemoveError = err => this._log.warn("Could not delete json file", err); - Async.promiseSpinningly( - Utils.jsonRemove("commands", this).catch(logRemoveError) - .then(Utils.jsonRemove("commands-syncing", this).catch(logRemoveError)) - ); }, removeClientData: function removeClientData() { @@ -471,6 +194,14 @@ ClientEngine.prototype = { }, /** + * Remove any commands for the local client and mark it for upload. + */ + clearCommands: function clearCommands() { + delete this.localCommands; + this._tracker.addChangedID(this.localID); + }, + + /** * Sends a command+args pair to a specific client. * * @param command Command string @@ -484,17 +215,30 @@ ClientEngine.prototype = { if (!client) { throw new Error("Unknown remote client ID: '" + clientId + "'."); } - if (client.stale) { - throw new Error("Stale remote client ID: '" + clientId + "'."); - } + + // notDupe compares two commands and returns if they are not equal. + let notDupe = function(other) { + return other.command != command || !Utils.deepEquals(other.args, args); + }; let action = { command: command, args: args, }; + if (!client.commands) { + client.commands = [action]; + } + // Add the new action if there are no duplicates. + else if (client.commands.every(notDupe)) { + client.commands.push(action); + } + // It must be a dupe. Skip. + else { + return; + } + this._log.trace("Client " + clientId + " got a new action: " + [command, args]); - this._addClientCommand(clientId, action); this._tracker.addChangedID(clientId); }, @@ -505,17 +249,13 @@ ClientEngine.prototype = { */ processIncomingCommands: function processIncomingCommands() { return this._notify("clients:process-commands", "", function() { - if (!this.localCommands) { - return true; - } + let commands = this.localCommands; - const clearedCommands = this._readCommands()[this.localID]; - const commands = this.localCommands.filter(command => !hasDupeCommand(clearedCommands, command)); + // Immediately clear out the commands as we've got them locally. + this.clearCommands(); - let URIsToDisplay = []; // Process each command in order. - for (let rawCommand of commands) { - let {command, args} = rawCommand; + for each (let {command, args} in commands) { this._log.debug("Processing command: " + command + "(" + args + ")"); let engines = [args[0]]; @@ -536,20 +276,12 @@ ClientEngine.prototype = { this.service.logout(); return false; case "displayURI": - let [uri, clientId, title] = args; - URIsToDisplay.push({ uri, clientId, title }); + this._handleDisplayURI.apply(this, args); break; default: this._log.debug("Received an unknown command: " + command); break; } - // Add the command to the "cleared" commands list - this._addClientCommand(this.localID, rawCommand) - } - this._tracker.addChangedID(this.localID); - - if (URIsToDisplay.length) { - this._handleDisplayURIs(URIsToDisplay); } return true; @@ -588,10 +320,8 @@ ClientEngine.prototype = { if (clientId) { this._sendCommandToClient(command, args, clientId); } else { - for (let [id, record] of Object.entries(this._store._remoteClients)) { - if (!record.stale) { - this._sendCommandToClient(command, args, id); - } + for (let id in this._store._remoteClients) { + this._sendCommandToClient(command, args, id); } } }, @@ -622,11 +352,11 @@ ClientEngine.prototype = { }, /** - * Handle a bunch of received 'displayURI' commands. + * Handle a single received 'displayURI' command. * - * Interested parties should observe the "weave:engine:clients:display-uris" - * topic. The callback will receive an array as the subject parameter - * containing objects with the following keys: + * Interested parties should observe the "weave:engine:clients:display-uri" + * topic. The callback will receive an object as the subject parameter with + * the following keys: * * uri URI (string) that is requested for display. * clientId ID of client that sent the command. @@ -634,24 +364,21 @@ ClientEngine.prototype = { * * The 'data' parameter to the callback will not be defined. * - * @param uris - * An array containing URI objects to display - * @param uris[].uri + * @param uri * String URI that was received - * @param uris[].clientId + * @param clientId * ID of client that sent URI - * @param uris[].title + * @param title * String title of page that URI corresponds to. Older clients may not * send this. */ - _handleDisplayURIs: function _handleDisplayURIs(uris) { - Svc.Obs.notify("weave:engine:clients:display-uris", uris); - }, + _handleDisplayURI: function _handleDisplayURI(uri, clientId, title) { + this._log.info("Received a URI for display: " + uri + " (" + title + + ") from " + clientId); - _removeRemoteClient(id) { - delete this._store._remoteClients[id]; - this._tracker.removeChangedID(id); - }, + let subject = {uri: uri, client: clientId, title: title}; + Svc.Obs.notify("weave:engine:clients:display-uri", subject); + } }; function ClientStore(name, engine) { @@ -660,48 +387,29 @@ function ClientStore(name, engine) { ClientStore.prototype = { __proto__: Store.prototype, - _remoteClients: {}, - create(record) { - this.update(record); + this.update(record) }, update: function update(record) { - if (record.id == this.engine.localID) { - // Only grab commands from the server; local name/type always wins + // Only grab commands from the server; local name/type always wins + if (record.id == this.engine.localID) this.engine.localCommands = record.commands; - } else { + else this._remoteClients[record.id] = record.cleartext; - } }, createRecord: function createRecord(id, collection) { let record = new ClientsRec(collection, id); - const commandsChanges = this.engine._currentlySyncingCommands ? - this.engine._currentlySyncingCommands[id] : - []; - // Package the individual components into a record for the local client if (id == this.engine.localID) { - let cb = Async.makeSpinningCallback(); - fxAccounts.getDeviceId().then(id => cb(null, id), cb); - try { - record.fxaDeviceId = cb.wait(); - } catch(error) { - this._log.warn("failed to get fxa device id", error); - } record.name = this.engine.localName; record.type = this.engine.localType; + record.commands = this.engine.localCommands; record.version = Services.appinfo.version; record.protocols = SUPPORTED_PROTOCOL_VERSIONS; - // Substract the commands we recorded that we've already executed - if (commandsChanges && commandsChanges.length && - this.engine.localCommands && this.engine.localCommands.length) { - record.commands = this.engine.localCommands.filter(command => !hasDupeCommand(commandsChanges, command)); - } - // Optional fields. record.os = Services.appinfo.OS; // "Darwin" record.appPackage = Services.appinfo.ID; @@ -712,20 +420,6 @@ ClientStore.prototype = { // record.formfactor = ""; // Bug 1100722 } else { record.cleartext = this._remoteClients[id]; - - // Add the commands we have to send - if (commandsChanges && commandsChanges.length) { - const recordCommands = record.cleartext.commands || []; - const newCommands = commandsChanges.filter(command => !hasDupeCommand(recordCommands, command)); - record.cleartext.commands = recordCommands.concat(newCommands); - } - - if (record.cleartext.stale) { - // It's almost certainly a logic error for us to upload a record we - // consider stale, so make log noise, but still remove the flag. - this._log.error(`Preparing to upload record ${id} that we consider stale`); - delete record.cleartext.stale; - } } return record; @@ -768,7 +462,7 @@ ClientsTracker.prototype = { break; case "weave:engine:stop-tracking": if (this._enabled) { - Svc.Prefs.ignore("client.name", this); + Svc.Prefs.ignore("clients.name", this); this._enabled = false; } break; diff --git a/services/sync/modules/engines/forms.js b/services/sync/modules/engines/forms.js index 43f79d4f7..11dd8d976 100644 --- a/services/sync/modules/engines/forms.js +++ b/services/sync/modules/engines/forms.js @@ -2,7 +2,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -this.EXPORTED_SYMBOLS = ['FormEngine', 'FormRec', 'FormValidator']; +this.EXPORTED_SYMBOLS = ['FormEngine', 'FormRec']; var Cc = Components.classes; var Ci = Components.interfaces; @@ -14,10 +14,9 @@ Cu.import("resource://services-sync/record.js"); Cu.import("resource://services-common/async.js"); Cu.import("resource://services-sync/util.js"); Cu.import("resource://services-sync/constants.js"); -Cu.import("resource://services-sync/collection_validator.js"); Cu.import("resource://gre/modules/Log.jsm"); -const FORMS_TTL = 3 * 365 * 24 * 60 * 60; // Three years in seconds. +const FORMS_TTL = 5184000; // 60 days this.FormRec = function FormRec(collection, id) { CryptoWrapper.call(this, collection, id); @@ -37,24 +36,20 @@ var FormWrapper = { _getEntryCols: ["fieldname", "value"], _guidCols: ["guid"], - _promiseSearch: function(terms, searchData) { - return new Promise(resolve => { - let results = []; - let callbacks = { - handleResult(result) { - results.push(result); - }, - handleCompletion(reason) { - resolve(results); - } - }; - Svc.FormHistory.search(terms, searchData, callbacks); - }) - }, - // Do a "sync" search by spinning the event loop until it completes. _searchSpinningly: function(terms, searchData) { - return Async.promiseSpinningly(this._promiseSearch(terms, searchData)); + let results = []; + let cb = Async.makeSpinningCallback(); + let callbacks = { + handleResult: function(result) { + results.push(result); + }, + handleCompletion: function(reason) { + cb(null, results); + } + }; + Svc.FormHistory.search(terms, searchData, callbacks); + return cb.wait(); }, _updateSpinningly: function(changes) { @@ -114,9 +109,7 @@ FormEngine.prototype = { syncPriority: 6, - get prefName() { - return "history"; - }, + get prefName() "history", _findDupe: function _findDupe(item) { return FormWrapper.getGUID(item.name, item.value); @@ -235,6 +228,7 @@ FormTracker.prototype = { if (this.ignoreAll) { return; } + switch (topic) { case "satchel-storage-changed": if (data == "formhistory-add" || data == "formhistory-remove") { @@ -250,56 +244,3 @@ FormTracker.prototype = { this.score += SCORE_INCREMENT_MEDIUM; }, }; - - -class FormsProblemData extends CollectionProblemData { - getSummary() { - // We don't support syncing deleted form data, so "clientMissing" isn't a problem - return super.getSummary().filter(entry => - entry.name !== "clientMissing"); - } -} - -class FormValidator extends CollectionValidator { - constructor() { - super("forms", "id", ["name", "value"]); - } - - emptyProblemData() { - return new FormsProblemData(); - } - - getClientItems() { - return FormWrapper._promiseSearch(["guid", "fieldname", "value"], {}); - } - - normalizeClientItem(item) { - return { - id: item.guid, - guid: item.guid, - name: item.fieldname, - fieldname: item.fieldname, - value: item.value, - original: item, - }; - } - - normalizeServerItem(item) { - let res = Object.assign({ - guid: item.id, - fieldname: item.name, - original: item, - }, item); - // Missing `name` or `value` causes the getGUID call to throw - if (item.name !== undefined && item.value !== undefined) { - let guid = FormWrapper.getGUID(item.name, item.value); - if (guid) { - res.guid = guid; - res.id = guid; - res.duped = true; - } - } - - return res; - } -}
\ No newline at end of file diff --git a/services/sync/modules/engines/history.js b/services/sync/modules/engines/history.js index 307d484c1..e7f53766f 100644 --- a/services/sync/modules/engines/history.js +++ b/services/sync/modules/engines/history.js @@ -44,25 +44,6 @@ HistoryEngine.prototype = { applyIncomingBatchSize: HISTORY_STORE_BATCH_SIZE, syncPriority: 7, - - _processIncoming: function (newitems) { - // We want to notify history observers that a batch operation is underway - // so they don't do lots of work for each incoming record. - let observers = PlacesUtils.history.getObservers(); - function notifyHistoryObservers(notification) { - for (let observer of observers) { - try { - observer[notification](); - } catch (ex) { } - } - } - notifyHistoryObservers("onBeginUpdateBatch"); - try { - return SyncEngine.prototype._processIncoming.call(this, newitems); - } finally { - notifyHistoryObservers("onEndUpdateBatch"); - } - }, }; function HistoryStore(name, engine) { @@ -105,7 +86,7 @@ HistoryStore.prototype = { return this._getStmt( "UPDATE moz_places " + "SET guid = :guid " + - "WHERE url_hash = hash(:page_url) AND url = :page_url"); + "WHERE url = :page_url"); }, // Some helper functions to handle GUIDs @@ -127,7 +108,7 @@ HistoryStore.prototype = { return this._getStmt( "SELECT guid " + "FROM moz_places " + - "WHERE url_hash = hash(:page_url) AND url = :page_url"); + "WHERE url = :page_url"); }, _guidCols: ["guid"], @@ -146,12 +127,12 @@ HistoryStore.prototype = { }, get _visitStm() { - return this._getStmt(`/* do not warn (bug 599936) */ - SELECT visit_type type, visit_date date - FROM moz_historyvisits - JOIN moz_places h ON h.id = place_id - WHERE url_hash = hash(:url) AND url = :url - ORDER BY date DESC LIMIT 20`); + return this._getStmt( + "/* do not warn (bug 599936) */ " + + "SELECT visit_type type, visit_date date " + + "FROM moz_historyvisits " + + "WHERE place_id = (SELECT id FROM moz_places WHERE url = :url) " + + "ORDER BY date DESC LIMIT 10"); }, _visitCols: ["date", "type"], @@ -223,10 +204,7 @@ HistoryStore.prototype = { } else { shouldApply = this._recordToPlaceInfo(record); } - } catch (ex) { - if (Async.isShutdownException(ex)) { - throw ex; - } + } catch(ex) { failed.push(record.id); shouldApply = false; } @@ -299,14 +277,14 @@ HistoryStore.prototype = { if (!visit.date || typeof visit.date != "number") { this._log.warn("Encountered record with invalid visit date: " + visit.date); - continue; + throw "Visit has no date!"; } - if (!visit.type || - !Object.values(PlacesUtils.history.TRANSITIONS).includes(visit.type)) { - this._log.warn("Encountered record with invalid visit type: " + - visit.type + "; ignoring."); - continue; + if (!visit.type || !(visit.type >= PlacesUtils.history.TRANSITION_LINK && + visit.type <= PlacesUtils.history.TRANSITION_RELOAD)) { + this._log.warn("Encountered record with invalid visit type: " + + visit.type); + throw "Invalid visit type!"; } // Dates need to be integers. @@ -317,7 +295,6 @@ HistoryStore.prototype = { // overwritten. continue; } - visit.visitDate = visit.date; visit.transitionType = visit.type; k += 1; @@ -369,9 +346,7 @@ HistoryStore.prototype = { }, wipe: function HistStore_wipe() { - let cb = Async.makeSyncCallback(); - PlacesUtils.history.clear().then(result => {cb(null, result)}, err => {cb(err)}); - return Async.waitForSyncCallback(cb); + PlacesUtils.history.removeAllPages(); } }; diff --git a/services/sync/modules/engines/passwords.js b/services/sync/modules/engines/passwords.js index 51db49a0a..0ccd2e7b0 100644 --- a/services/sync/modules/engines/passwords.js +++ b/services/sync/modules/engines/passwords.js @@ -2,16 +2,14 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -this.EXPORTED_SYMBOLS = ['PasswordEngine', 'LoginRec', 'PasswordValidator']; +this.EXPORTED_SYMBOLS = ['PasswordEngine', 'LoginRec']; var {classes: Cc, interfaces: Ci, utils: Cu} = Components; Cu.import("resource://services-sync/record.js"); Cu.import("resource://services-sync/constants.js"); -Cu.import("resource://services-sync/collection_validator.js"); Cu.import("resource://services-sync/engines.js"); Cu.import("resource://services-sync/util.js"); -Cu.import("resource://services-common/async.js"); this.LoginRec = function LoginRec(collection, id) { CryptoWrapper.call(this, collection, id); @@ -24,7 +22,6 @@ LoginRec.prototype = { Utils.deferGetSet(LoginRec, "cleartext", [ "hostname", "formSubmitURL", "httpRealm", "username", "password", "usernameField", "passwordField", - "timeCreated", "timePasswordChanged", ]); @@ -70,10 +67,7 @@ PasswordEngine.prototype = { Svc.Prefs.set("deletePwdFxA", true); Svc.Prefs.reset("deletePwd"); // The old prefname we previously used. } catch (ex) { - if (Async.isShutdownException(ex)) { - throw ex; - } - this._log.debug("Password deletes failed", ex); + this._log.debug("Password deletes failed: ", ex); } } }, @@ -104,13 +98,6 @@ function PasswordStore(name, engine) { PasswordStore.prototype = { __proto__: Store.prototype, - _newPropertyBag: function () { - return Cc["@mozilla.org/hash-property-bag;1"].createInstance(Ci.nsIWritablePropertyBag2); - }, - - /** - * Return an instance of nsILoginInfo (and, implicitly, nsILoginMetaInfo). - */ _nsLoginInfoFromRecord: function (record) { function nullUndefined(x) { return (x == undefined) ? null : x; @@ -131,21 +118,13 @@ PasswordStore.prototype = { record.password, record.usernameField, record.passwordField); - info.QueryInterface(Ci.nsILoginMetaInfo); info.guid = record.id; - if (record.timeCreated) { - info.timeCreated = record.timeCreated; - } - if (record.timePasswordChanged) { - info.timePasswordChanged = record.timePasswordChanged; - } - return info; }, _getLoginFromGUID: function (id) { - let prop = this._newPropertyBag(); + let prop = Cc["@mozilla.org/hash-property-bag;1"].createInstance(Ci.nsIWritablePropertyBag2); prop.setPropertyAsAUTF8String("guid", id); let logins = Services.logins.searchLogins({}, prop); @@ -190,7 +169,8 @@ PasswordStore.prototype = { return; } - let prop = this._newPropertyBag(); + let prop = Cc["@mozilla.org/hash-property-bag;1"] + .createInstance(Ci.nsIWritablePropertyBag2); prop.setPropertyAsAUTF8String("guid", newID); Services.logins.modifyLogin(oldLogin, prop); @@ -217,11 +197,6 @@ PasswordStore.prototype = { record.usernameField = login.usernameField; record.passwordField = login.passwordField; - // Optional fields. - login.QueryInterface(Ci.nsILoginMetaInfo); - record.timeCreated = login.timeCreated; - record.timePasswordChanged = login.timePasswordChanged; - return record; }, @@ -237,7 +212,8 @@ PasswordStore.prototype = { try { Services.logins.addLogin(login); } catch(ex) { - this._log.debug(`Adding record ${record.id} resulted in exception`, ex); + this._log.debug("Adding record " + record.id + + " resulted in exception ", ex); } }, @@ -269,7 +245,8 @@ PasswordStore.prototype = { try { Services.logins.modifyLogin(loginItem, newinfo); } catch(ex) { - this._log.debug(`Modifying record ${record.id} resulted in exception; not modifying`, ex); + this._log.debug("Modifying record " + record.id + + " resulted in exception. Not modifying.", ex); } }, @@ -326,46 +303,3 @@ PasswordTracker.prototype = { } }, }; - -class PasswordValidator extends CollectionValidator { - constructor() { - super("passwords", "id", [ - "hostname", - "formSubmitURL", - "httpRealm", - "password", - "passwordField", - "username", - "usernameField", - ]); - } - - getClientItems() { - let logins = Services.logins.getAllLogins({}); - let syncHosts = Utils.getSyncCredentialsHosts() - let result = logins.map(l => l.QueryInterface(Ci.nsILoginMetaInfo)) - .filter(l => !syncHosts.has(l.hostname)); - return Promise.resolve(result); - } - - normalizeClientItem(item) { - return { - id: item.guid, - guid: item.guid, - hostname: item.hostname, - formSubmitURL: item.formSubmitURL, - httpRealm: item.httpRealm, - password: item.password, - passwordField: item.passwordField, - username: item.username, - usernameField: item.usernameField, - original: item, - } - } - - normalizeServerItem(item) { - return Object.assign({ guid: item.id }, item); - } -} - - diff --git a/services/sync/modules/engines/prefs.js b/services/sync/modules/engines/prefs.js index 9ceeb9ac6..792e0c66a 100644 --- a/services/sync/modules/engines/prefs.js +++ b/services/sync/modules/engines/prefs.js @@ -8,7 +8,7 @@ var Cc = Components.classes; var Ci = Components.interfaces; var Cu = Components.utils; -const PREF_SYNC_PREFS_PREFIX = "services.sync.prefs.sync."; +const SYNC_PREFS_PREFIX = "services.sync.prefs.sync."; Cu.import("resource://services-sync/engines.js"); Cu.import("resource://services-sync/record.js"); @@ -42,7 +42,6 @@ PrefsEngine.prototype = { version: 2, syncPriority: 1, - allowSkippedRecord: false, getChangedIDs: function () { // No need for a proper timestamp (no conflict resolution needed). @@ -88,45 +87,37 @@ PrefStore.prototype = { _getSyncPrefs: function () { let syncPrefs = Cc["@mozilla.org/preferences-service;1"] .getService(Ci.nsIPrefService) - .getBranch(PREF_SYNC_PREFS_PREFIX) + .getBranch(SYNC_PREFS_PREFIX) .getChildList("", {}); // Also sync preferences that determine which prefs get synced. - let controlPrefs = syncPrefs.map(pref => PREF_SYNC_PREFS_PREFIX + pref); + let controlPrefs = syncPrefs.map(pref => SYNC_PREFS_PREFIX + pref); return controlPrefs.concat(syncPrefs); }, _isSynced: function (pref) { - return pref.startsWith(PREF_SYNC_PREFS_PREFIX) || - this._prefs.get(PREF_SYNC_PREFS_PREFIX + pref, false); + return pref.startsWith(SYNC_PREFS_PREFIX) || + this._prefs.get(SYNC_PREFS_PREFIX + pref, false); }, _getAllPrefs: function () { let values = {}; - for (let pref of this._getSyncPrefs()) { + for each (let pref in this._getSyncPrefs()) { if (this._isSynced(pref)) { - // Missing and default prefs get the null value. - values[pref] = this._prefs.isSet(pref) ? this._prefs.get(pref, null) : null; + // Missing prefs get the null value. + values[pref] = this._prefs.get(pref, null); } } return values; }, - _updateLightWeightTheme (themeID) { - let themeObject = null; - if (themeID) { - themeObject = LightweightThemeManager.getUsedTheme(themeID); - } - LightweightThemeManager.currentTheme = themeObject; - }, - _setAllPrefs: function (values) { - let selectedThemeIDPref = "lightweightThemes.selectedThemeID"; - let selectedThemeIDBefore = this._prefs.get(selectedThemeIDPref, null); - let selectedThemeIDAfter = selectedThemeIDBefore; + let enabledPref = "lightweightThemes.isThemeSelected"; + let enabledBefore = this._prefs.get(enabledPref, false); + let prevTheme = LightweightThemeManager.currentTheme; // Update 'services.sync.prefs.sync.foo.pref' before 'foo.pref', otherwise // _isSynced returns false when 'foo.pref' doesn't exist (e.g., on a new device). - let prefs = Object.keys(values).sort(a => -a.indexOf(PREF_SYNC_PREFS_PREFIX)); + let prefs = Object.keys(values).sort(a => -a.indexOf(SYNC_PREFS_PREFIX)); for (let pref of prefs) { if (!this._isSynced(pref)) { continue; @@ -134,30 +125,26 @@ PrefStore.prototype = { let value = values[pref]; - switch (pref) { - // Some special prefs we don't want to set directly. - case selectedThemeIDPref: - selectedThemeIDAfter = value; - break; - - // default is to just set the pref - default: - if (value == null) { - // Pref has gone missing. The best we can do is reset it. - this._prefs.reset(pref); - } else { - try { - this._prefs.set(pref, value); - } catch(ex) { - this._log.trace("Failed to set pref: " + pref + ": " + ex); - } - } + // Pref has gone missing. The best we can do is reset it. + if (value == null) { + this._prefs.reset(pref); + continue; } + + try { + this._prefs.set(pref, value); + } catch(ex) { + this._log.trace("Failed to set pref: " + pref + ": " + ex); + } } - // Notify the lightweight theme manager if the selected theme has changed. - if (selectedThemeIDBefore != selectedThemeIDAfter) { - this._updateLightWeightTheme(selectedThemeIDAfter); + // Notify the lightweight theme manager of all the new values + let enabledNow = this._prefs.get(enabledPref, false); + if (enabledBefore && !enabledNow) { + LightweightThemeManager.currentTheme = null; + } else if (enabledNow && LightweightThemeManager.usedThemes[0] != prevTheme) { + LightweightThemeManager.currentTheme = null; + LightweightThemeManager.currentTheme = LightweightThemeManager.usedThemes[0]; } }, @@ -261,8 +248,8 @@ PrefTracker.prototype = { case "nsPref:changed": // Trigger a sync for MULTI-DEVICE for a change that determines // which prefs are synced or a regular pref change. - if (data.indexOf(PREF_SYNC_PREFS_PREFIX) == 0 || - this._prefs.get(PREF_SYNC_PREFS_PREFIX + data, false)) { + if (data.indexOf(SYNC_PREFS_PREFIX) == 0 || + this._prefs.get(SYNC_PREFS_PREFIX + data, false)) { this.score += SCORE_INCREMENT_XLARGE; this.modified = true; this._log.trace("Preference " + data + " changed"); diff --git a/services/sync/modules/engines/tabs.js b/services/sync/modules/engines/tabs.js index 45ece4a23..167faf625 100644 --- a/services/sync/modules/engines/tabs.js +++ b/services/sync/modules/engines/tabs.js @@ -43,11 +43,6 @@ TabEngine.prototype = { _storeObj: TabStore, _trackerObj: TabTracker, _recordObj: TabSetRecord, - // A flag to indicate if we have synced in this session. This is to help - // consumers of remote tabs that may want to differentiate between "I've an - // empty tab list as I haven't yet synced" vs "I've an empty tab list - // as there really are no tabs" - hasSyncedThisSession: false, syncPriority: 3, @@ -72,7 +67,6 @@ TabEngine.prototype = { SyncEngine.prototype._resetClient.call(this); this._store.wipe(); this._tracker.modified = true; - this.hasSyncedThisSession = false; }, removeClientData: function () { @@ -100,12 +94,7 @@ TabEngine.prototype = { } return SyncEngine.prototype._reconcile.call(this, item); - }, - - _syncFinish() { - this.hasSyncedThisSession = true; - return SyncEngine.prototype._syncFinish.call(this); - }, + } }; @@ -145,7 +134,7 @@ TabStore.prototype = { } for (let tab of win.gBrowser.tabs) { - let tabState = this.getTabState(tab); + tabState = this.getTabState(tab); // Make sure there are history entries to look at. if (!tabState || !tabState.entries.length) { @@ -165,11 +154,6 @@ TabStore.prototype = { continue; } - if (current.url.length >= (MAX_UPLOAD_BYTES - 1000)) { - this._log.trace("Skipping over-long URL."); - continue; - } - // The element at `index` is the current page. Previous URLs were // previously visited URLs; subsequent URLs are in the 'forward' stack, // which we can't represent in Sync, so we truncate here. @@ -189,9 +173,7 @@ TabStore.prototype = { allTabs.push({ title: current.title || "", urlHistory: urls, - icon: tabState.image || - (tabState.attributes && tabState.attributes.image) || - "", + icon: tabState.attributes && tabState.attributes.image || "", lastUsed: Math.floor((tabState.lastAccessed || 0) / 1000), }); } @@ -265,9 +247,27 @@ TabStore.prototype = { create: function (record) { this._log.debug("Adding remote tabs from " + record.clientName); - this._remoteClients[record.id] = Object.assign({}, record.cleartext, { - lastModified: record.modified - }); + this._remoteClients[record.id] = record.cleartext; + + // Lose some precision, but that's good enough (seconds). + let roundModify = Math.floor(record.modified / 1000); + let notifyState = Svc.Prefs.get("notifyTabState"); + + // If there's no existing pref, save this first modified time. + if (notifyState == null) { + Svc.Prefs.set("notifyTabState", roundModify); + return; + } + + // Don't change notifyState if it's already 0 (don't notify). + if (notifyState == 0) { + return; + } + + // We must have gotten a new tab that isn't the same as last time. + if (notifyState != roundModify) { + Svc.Prefs.set("notifyTabState", 0); + } }, update: function (record) { @@ -306,10 +306,6 @@ TabTracker.prototype = { window.addEventListener(topic, this.onTab, false); } window.addEventListener("unload", this._unregisterListeners, false); - // If it's got a tab browser we can listen for things like navigation. - if (window.gBrowser) { - window.gBrowser.addProgressListener(this); - } }, _unregisterListeners: function (event) { @@ -322,9 +318,6 @@ TabTracker.prototype = { for (let topic of this._topics) { window.removeEventListener(topic, this.onTab, false); } - if (window.gBrowser) { - window.gBrowser.removeProgressListener(this); - } }, startTracking: function () { @@ -380,14 +373,4 @@ TabTracker.prototype = { this.score += SCORE_INCREMENT_SMALL; } }, - - // web progress listeners. - onLocationChange: function (webProgress, request, location, flags) { - // We only care about top-level location changes which are not in the same - // document. - if (webProgress.isTopLevel && - ((flags & Ci.nsIWebProgressListener.LOCATION_CHANGE_SAME_DOCUMENT) == 0)) { - this.modified = true; - } - }, }; |