summaryrefslogtreecommitdiffstats
path: root/services/sync/modules/engines
diff options
context:
space:
mode:
Diffstat (limited to 'services/sync/modules/engines')
-rw-r--r--services/sync/modules/engines/addons.js813
-rw-r--r--services/sync/modules/engines/bookmarks.js1378
-rw-r--r--services/sync/modules/engines/clients.js782
-rw-r--r--services/sync/modules/engines/extension-storage.js277
-rw-r--r--services/sync/modules/engines/forms.js305
-rw-r--r--services/sync/modules/engines/history.js442
-rw-r--r--services/sync/modules/engines/passwords.js371
-rw-r--r--services/sync/modules/engines/prefs.js273
-rw-r--r--services/sync/modules/engines/tabs.js393
9 files changed, 5034 insertions, 0 deletions
diff --git a/services/sync/modules/engines/addons.js b/services/sync/modules/engines/addons.js
new file mode 100644
index 000000000..01dab58d1
--- /dev/null
+++ b/services/sync/modules/engines/addons.js
@@ -0,0 +1,813 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/*
+ * This file defines the add-on sync functionality.
+ *
+ * There are currently a number of known limitations:
+ * - We only sync XPI extensions and themes available from addons.mozilla.org.
+ * We hope to expand support for other add-ons eventually.
+ * - We only attempt syncing of add-ons between applications of the same type.
+ * This means add-ons will not synchronize between Firefox desktop and
+ * Firefox mobile, for example. This is because of significant add-on
+ * incompatibility between application types.
+ *
+ * Add-on records exist for each known {add-on, app-id} pair in the Sync client
+ * set. Each record has a randomly chosen GUID. The records then contain
+ * basic metadata about the add-on.
+ *
+ * We currently synchronize:
+ *
+ * - Installations
+ * - Uninstallations
+ * - User enabling and disabling
+ *
+ * Synchronization is influenced by the following preferences:
+ *
+ * - services.sync.addons.ignoreUserEnabledChanges
+ * - services.sync.addons.trustedSourceHostnames
+ *
+ * and also influenced by whether addons have repository caching enabled and
+ * whether they allow installation of addons from insecure options (both of
+ * which are themselves influenced by the "extensions." pref branch)
+ *
+ * See the documentation in services-sync.js for the behavior of these prefs.
+ */
+"use strict";
+
+var {classes: Cc, interfaces: Ci, utils: Cu} = Components;
+
+Cu.import("resource://services-sync/addonutils.js");
+Cu.import("resource://services-sync/addonsreconciler.js");
+Cu.import("resource://services-sync/engines.js");
+Cu.import("resource://services-sync/record.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://services-sync/collection_validator.js");
+Cu.import("resource://services-common/async.js");
+
+Cu.import("resource://gre/modules/Preferences.jsm");
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "AddonManager",
+ "resource://gre/modules/AddonManager.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "AddonRepository",
+ "resource://gre/modules/addons/AddonRepository.jsm");
+
+this.EXPORTED_SYMBOLS = ["AddonsEngine", "AddonValidator"];
+
+// 7 days in milliseconds.
+const PRUNE_ADDON_CHANGES_THRESHOLD = 60 * 60 * 24 * 7 * 1000;
+
+/**
+ * AddonRecord represents the state of an add-on in an application.
+ *
+ * Each add-on has its own record for each application ID it is installed
+ * on.
+ *
+ * The ID of add-on records is a randomly-generated GUID. It is random instead
+ * of deterministic so the URIs of the records cannot be guessed and so
+ * compromised server credentials won't result in disclosure of the specific
+ * add-ons present in a Sync account.
+ *
+ * The record contains the following fields:
+ *
+ * addonID
+ * ID of the add-on. This correlates to the "id" property on an Addon type.
+ *
+ * applicationID
+ * The application ID this record is associated with.
+ *
+ * enabled
+ * Boolean stating whether add-on is enabled or disabled by the user.
+ *
+ * source
+ * String indicating where an add-on is from. Currently, we only support
+ * the value "amo" which indicates that the add-on came from the official
+ * add-ons repository, addons.mozilla.org. In the future, we may support
+ * installing add-ons from other sources. This provides a future-compatible
+ * mechanism for clients to only apply records they know how to handle.
+ */
+function AddonRecord(collection, id) {
+ CryptoWrapper.call(this, collection, id);
+}
+AddonRecord.prototype = {
+ __proto__: CryptoWrapper.prototype,
+ _logName: "Record.Addon"
+};
+
+Utils.deferGetSet(AddonRecord, "cleartext", ["addonID",
+ "applicationID",
+ "enabled",
+ "source"]);
+
+/**
+ * The AddonsEngine handles synchronization of add-ons between clients.
+ *
+ * The engine maintains an instance of an AddonsReconciler, which is the entity
+ * maintaining state for add-ons. It provides the history and tracking APIs
+ * that AddonManager doesn't.
+ *
+ * The engine instance overrides a handful of functions on the base class. The
+ * rationale for each is documented by that function.
+ */
+this.AddonsEngine = function AddonsEngine(service) {
+ SyncEngine.call(this, "Addons", service);
+
+ this._reconciler = new AddonsReconciler();
+}
+AddonsEngine.prototype = {
+ __proto__: SyncEngine.prototype,
+ _storeObj: AddonsStore,
+ _trackerObj: AddonsTracker,
+ _recordObj: AddonRecord,
+ version: 1,
+
+ syncPriority: 5,
+
+ _reconciler: null,
+
+ /**
+ * Override parent method to find add-ons by their public ID, not Sync GUID.
+ */
+ _findDupe: function _findDupe(item) {
+ let id = item.addonID;
+
+ // The reconciler should have been updated at the top of the sync, so we
+ // can assume it is up to date when this function is called.
+ let addons = this._reconciler.addons;
+ if (!(id in addons)) {
+ return null;
+ }
+
+ let addon = addons[id];
+ if (addon.guid != item.id) {
+ return addon.guid;
+ }
+
+ return null;
+ },
+
+ /**
+ * Override getChangedIDs to pull in tracker changes plus changes from the
+ * reconciler log.
+ */
+ getChangedIDs: function getChangedIDs() {
+ let changes = {};
+ for (let [id, modified] of Object.entries(this._tracker.changedIDs)) {
+ changes[id] = modified;
+ }
+
+ let lastSyncDate = new Date(this.lastSync * 1000);
+
+ // The reconciler should have been refreshed at the beginning of a sync and
+ // we assume this function is only called from within a sync.
+ let reconcilerChanges = this._reconciler.getChangesSinceDate(lastSyncDate);
+ let addons = this._reconciler.addons;
+ for (let change of reconcilerChanges) {
+ let changeTime = change[0];
+ let id = change[2];
+
+ if (!(id in addons)) {
+ continue;
+ }
+
+ // Keep newest modified time.
+ if (id in changes && changeTime < changes[id]) {
+ continue;
+ }
+
+ if (!this.isAddonSyncable(addons[id])) {
+ continue;
+ }
+
+ this._log.debug("Adding changed add-on from changes log: " + id);
+ let addon = addons[id];
+ changes[addon.guid] = changeTime.getTime() / 1000;
+ }
+
+ return changes;
+ },
+
+ /**
+ * Override start of sync function to refresh reconciler.
+ *
+ * Many functions in this class assume the reconciler is refreshed at the
+ * top of a sync. If this ever changes, those functions should be revisited.
+ *
+ * Technically speaking, we don't need to refresh the reconciler on every
+ * sync since it is installed as an AddonManager listener. However, add-ons
+ * are complicated and we force a full refresh, just in case the listeners
+ * missed something.
+ */
+ _syncStartup: function _syncStartup() {
+ // We refresh state before calling parent because syncStartup in the parent
+ // looks for changed IDs, which is dependent on add-on state being up to
+ // date.
+ this._refreshReconcilerState();
+
+ SyncEngine.prototype._syncStartup.call(this);
+ },
+
+ /**
+ * Override end of sync to perform a little housekeeping on the reconciler.
+ *
+ * We prune changes to prevent the reconciler state from growing without
+ * bound. Even if it grows unbounded, there would have to be many add-on
+ * changes (thousands) for it to slow things down significantly. This is
+ * highly unlikely to occur. Still, we exercise defense just in case.
+ */
+ _syncCleanup: function _syncCleanup() {
+ let ms = 1000 * this.lastSync - PRUNE_ADDON_CHANGES_THRESHOLD;
+ this._reconciler.pruneChangesBeforeDate(new Date(ms));
+
+ SyncEngine.prototype._syncCleanup.call(this);
+ },
+
+ /**
+ * Helper function to ensure reconciler is up to date.
+ *
+ * This will synchronously load the reconciler's state from the file
+ * system (if needed) and refresh the state of the reconciler.
+ */
+ _refreshReconcilerState: function _refreshReconcilerState() {
+ this._log.debug("Refreshing reconciler state");
+ let cb = Async.makeSpinningCallback();
+ this._reconciler.refreshGlobalState(cb);
+ cb.wait();
+ },
+
+ isAddonSyncable(addon, ignoreRepoCheck) {
+ return this._store.isAddonSyncable(addon, ignoreRepoCheck);
+ }
+};
+
+/**
+ * This is the primary interface between Sync and the Addons Manager.
+ *
+ * In addition to the core store APIs, we provide convenience functions to wrap
+ * Add-on Manager APIs with Sync-specific semantics.
+ */
+function AddonsStore(name, engine) {
+ Store.call(this, name, engine);
+}
+AddonsStore.prototype = {
+ __proto__: Store.prototype,
+
+ // Define the add-on types (.type) that we support.
+ _syncableTypes: ["extension", "theme"],
+
+ _extensionsPrefs: new Preferences("extensions."),
+
+ get reconciler() {
+ return this.engine._reconciler;
+ },
+
+ /**
+ * Override applyIncoming to filter out records we can't handle.
+ */
+ applyIncoming: function applyIncoming(record) {
+ // The fields we look at aren't present when the record is deleted.
+ if (!record.deleted) {
+ // Ignore records not belonging to our application ID because that is the
+ // current policy.
+ if (record.applicationID != Services.appinfo.ID) {
+ this._log.info("Ignoring incoming record from other App ID: " +
+ record.id);
+ return;
+ }
+
+ // Ignore records that aren't from the official add-on repository, as that
+ // is our current policy.
+ if (record.source != "amo") {
+ this._log.info("Ignoring unknown add-on source (" + record.source + ")" +
+ " for " + record.id);
+ return;
+ }
+ }
+
+ // Ignore incoming records for which an existing non-syncable addon
+ // exists.
+ let existingMeta = this.reconciler.addons[record.addonID];
+ if (existingMeta && !this.isAddonSyncable(existingMeta)) {
+ this._log.info("Ignoring incoming record for an existing but non-syncable addon", record.addonID);
+ return;
+ }
+
+ Store.prototype.applyIncoming.call(this, record);
+ },
+
+
+ /**
+ * Provides core Store API to create/install an add-on from a record.
+ */
+ create: function create(record) {
+ let cb = Async.makeSpinningCallback();
+ AddonUtils.installAddons([{
+ id: record.addonID,
+ syncGUID: record.id,
+ enabled: record.enabled,
+ requireSecureURI: this._extensionsPrefs.get("install.requireSecureOrigin", true),
+ }], cb);
+
+ // This will throw if there was an error. This will get caught by the sync
+ // engine and the record will try to be applied later.
+ let results = cb.wait();
+
+ if (results.skipped.includes(record.addonID)) {
+ this._log.info("Add-on skipped: " + record.addonID);
+ // Just early-return for skipped addons - we don't want to arrange to
+ // try again next time because the condition that caused up to skip
+ // will remain true for this addon forever.
+ return;
+ }
+
+ let addon;
+ for (let a of results.addons) {
+ if (a.id == record.addonID) {
+ addon = a;
+ break;
+ }
+ }
+
+ // This should never happen, but is present as a fail-safe.
+ if (!addon) {
+ throw new Error("Add-on not found after install: " + record.addonID);
+ }
+
+ this._log.info("Add-on installed: " + record.addonID);
+ },
+
+ /**
+ * Provides core Store API to remove/uninstall an add-on from a record.
+ */
+ remove: function remove(record) {
+ // If this is called, the payload is empty, so we have to find by GUID.
+ let addon = this.getAddonByGUID(record.id);
+ if (!addon) {
+ // We don't throw because if the add-on could not be found then we assume
+ // it has already been uninstalled and there is nothing for this function
+ // to do.
+ return;
+ }
+
+ this._log.info("Uninstalling add-on: " + addon.id);
+ let cb = Async.makeSpinningCallback();
+ AddonUtils.uninstallAddon(addon, cb);
+ cb.wait();
+ },
+
+ /**
+ * Provides core Store API to update an add-on from a record.
+ */
+ update: function update(record) {
+ let addon = this.getAddonByID(record.addonID);
+
+ // update() is called if !this.itemExists. And, since itemExists consults
+ // the reconciler only, we need to take care of some corner cases.
+ //
+ // First, the reconciler could know about an add-on that was uninstalled
+ // and no longer present in the add-ons manager.
+ if (!addon) {
+ this.create(record);
+ return;
+ }
+
+ // It's also possible that the add-on is non-restartless and has pending
+ // install/uninstall activity.
+ //
+ // We wouldn't get here if the incoming record was for a deletion. So,
+ // check for pending uninstall and cancel if necessary.
+ if (addon.pendingOperations & AddonManager.PENDING_UNINSTALL) {
+ addon.cancelUninstall();
+
+ // We continue with processing because there could be state or ID change.
+ }
+
+ let cb = Async.makeSpinningCallback();
+ this.updateUserDisabled(addon, !record.enabled, cb);
+ cb.wait();
+ },
+
+ /**
+ * Provide core Store API to determine if a record exists.
+ */
+ itemExists: function itemExists(guid) {
+ let addon = this.reconciler.getAddonStateFromSyncGUID(guid);
+
+ return !!addon;
+ },
+
+ /**
+ * Create an add-on record from its GUID.
+ *
+ * @param guid
+ * Add-on GUID (from extensions DB)
+ * @param collection
+ * Collection to add record to.
+ *
+ * @return AddonRecord instance
+ */
+ createRecord: function createRecord(guid, collection) {
+ let record = new AddonRecord(collection, guid);
+ record.applicationID = Services.appinfo.ID;
+
+ let addon = this.reconciler.getAddonStateFromSyncGUID(guid);
+
+ // If we don't know about this GUID or if it has been uninstalled, we mark
+ // the record as deleted.
+ if (!addon || !addon.installed) {
+ record.deleted = true;
+ return record;
+ }
+
+ record.modified = addon.modified.getTime() / 1000;
+
+ record.addonID = addon.id;
+ record.enabled = addon.enabled;
+
+ // This needs to be dynamic when add-ons don't come from AddonRepository.
+ record.source = "amo";
+
+ return record;
+ },
+
+ /**
+ * Changes the id of an add-on.
+ *
+ * This implements a core API of the store.
+ */
+ changeItemID: function changeItemID(oldID, newID) {
+ // We always update the GUID in the reconciler because it will be
+ // referenced later in the sync process.
+ let state = this.reconciler.getAddonStateFromSyncGUID(oldID);
+ if (state) {
+ state.guid = newID;
+ let cb = Async.makeSpinningCallback();
+ this.reconciler.saveState(null, cb);
+ cb.wait();
+ }
+
+ let addon = this.getAddonByGUID(oldID);
+ if (!addon) {
+ this._log.debug("Cannot change item ID (" + oldID + ") in Add-on " +
+ "Manager because old add-on not present: " + oldID);
+ return;
+ }
+
+ addon.syncGUID = newID;
+ },
+
+ /**
+ * Obtain the set of all syncable add-on Sync GUIDs.
+ *
+ * This implements a core Store API.
+ */
+ getAllIDs: function getAllIDs() {
+ let ids = {};
+
+ let addons = this.reconciler.addons;
+ for (let id in addons) {
+ let addon = addons[id];
+ if (this.isAddonSyncable(addon)) {
+ ids[addon.guid] = true;
+ }
+ }
+
+ return ids;
+ },
+
+ /**
+ * Wipe engine data.
+ *
+ * This uninstalls all syncable addons from the application. In case of
+ * error, it logs the error and keeps trying with other add-ons.
+ */
+ wipe: function wipe() {
+ this._log.info("Processing wipe.");
+
+ this.engine._refreshReconcilerState();
+
+ // We only wipe syncable add-ons. Wipe is a Sync feature not a security
+ // feature.
+ for (let guid in this.getAllIDs()) {
+ let addon = this.getAddonByGUID(guid);
+ if (!addon) {
+ this._log.debug("Ignoring add-on because it couldn't be obtained: " +
+ guid);
+ continue;
+ }
+
+ this._log.info("Uninstalling add-on as part of wipe: " + addon.id);
+ Utils.catch.call(this, () => addon.uninstall())();
+ }
+ },
+
+ /***************************************************************************
+ * Functions below are unique to this store and not part of the Store API *
+ ***************************************************************************/
+
+ /**
+ * Synchronously obtain an add-on from its public ID.
+ *
+ * @param id
+ * Add-on ID
+ * @return Addon or undefined if not found
+ */
+ getAddonByID: function getAddonByID(id) {
+ let cb = Async.makeSyncCallback();
+ AddonManager.getAddonByID(id, cb);
+ return Async.waitForSyncCallback(cb);
+ },
+
+ /**
+ * Synchronously obtain an add-on from its Sync GUID.
+ *
+ * @param guid
+ * Add-on Sync GUID
+ * @return DBAddonInternal or null
+ */
+ getAddonByGUID: function getAddonByGUID(guid) {
+ let cb = Async.makeSyncCallback();
+ AddonManager.getAddonBySyncGUID(guid, cb);
+ return Async.waitForSyncCallback(cb);
+ },
+
+ /**
+ * Determines whether an add-on is suitable for Sync.
+ *
+ * @param addon
+ * Addon instance
+ * @param ignoreRepoCheck
+ * Should we skip checking the Addons repository (primarially useful
+ * for testing and validation).
+ * @return Boolean indicating whether it is appropriate for Sync
+ */
+ isAddonSyncable: function isAddonSyncable(addon, ignoreRepoCheck = false) {
+ // Currently, we limit syncable add-ons to those that are:
+ // 1) In a well-defined set of types
+ // 2) Installed in the current profile
+ // 3) Not installed by a foreign entity (i.e. installed by the app)
+ // since they act like global extensions.
+ // 4) Is not a hotfix.
+ // 5) The addons XPIProvider doesn't veto it (i.e not being installed in
+ // the profile directory, or any other reasons it says the addon can't
+ // be synced)
+ // 6) Are installed from AMO
+
+ // We could represent the test as a complex boolean expression. We go the
+ // verbose route so the failure reason is logged.
+ if (!addon) {
+ this._log.debug("Null object passed to isAddonSyncable.");
+ return false;
+ }
+
+ if (this._syncableTypes.indexOf(addon.type) == -1) {
+ this._log.debug(addon.id + " not syncable: type not in whitelist: " +
+ addon.type);
+ return false;
+ }
+
+ if (!(addon.scope & AddonManager.SCOPE_PROFILE)) {
+ this._log.debug(addon.id + " not syncable: not installed in profile.");
+ return false;
+ }
+
+ // If the addon manager says it's not syncable, we skip it.
+ if (!addon.isSyncable) {
+ this._log.debug(addon.id + " not syncable: vetoed by the addon manager.");
+ return false;
+ }
+
+ // This may be too aggressive. If an add-on is downloaded from AMO and
+ // manually placed in the profile directory, foreignInstall will be set.
+ // Arguably, that add-on should be syncable.
+ // TODO Address the edge case and come up with more robust heuristics.
+ if (addon.foreignInstall) {
+ this._log.debug(addon.id + " not syncable: is foreign install.");
+ return false;
+ }
+
+ // Ignore hotfix extensions (bug 741670). The pref may not be defined.
+ // XXX - note that addon.isSyncable will be false for hotfix addons, so
+ // this check isn't strictly necessary - except for Sync tests which aren't
+ // setup to create a "real" hotfix addon. This can be removed once those
+ // tests are fixed (but keeping it doesn't hurt either)
+ if (this._extensionsPrefs.get("hotfix.id", null) == addon.id) {
+ this._log.debug(addon.id + " not syncable: is a hotfix.");
+ return false;
+ }
+
+ // If the AddonRepository's cache isn't enabled (which it typically isn't
+ // in tests), getCachedAddonByID always returns null - so skip the check
+ // in that case. We also provide a way to specifically opt-out of the check
+ // even if the cache is enabled, which is used by the validators.
+ if (ignoreRepoCheck || !AddonRepository.cacheEnabled) {
+ return true;
+ }
+
+ let cb = Async.makeSyncCallback();
+ AddonRepository.getCachedAddonByID(addon.id, cb);
+ let result = Async.waitForSyncCallback(cb);
+
+ if (!result) {
+ this._log.debug(addon.id + " not syncable: add-on not found in add-on " +
+ "repository.");
+ return false;
+ }
+
+ return this.isSourceURITrusted(result.sourceURI);
+ },
+
+ /**
+ * Determine whether an add-on's sourceURI field is trusted and the add-on
+ * can be installed.
+ *
+ * This function should only ever be called from isAddonSyncable(). It is
+ * exposed as a separate function to make testing easier.
+ *
+ * @param uri
+ * nsIURI instance to validate
+ * @return bool
+ */
+ isSourceURITrusted: function isSourceURITrusted(uri) {
+ // For security reasons, we currently limit synced add-ons to those
+ // installed from trusted hostname(s). We additionally require TLS with
+ // the add-ons site to help prevent forgeries.
+ let trustedHostnames = Svc.Prefs.get("addons.trustedSourceHostnames", "")
+ .split(",");
+
+ if (!uri) {
+ this._log.debug("Undefined argument to isSourceURITrusted().");
+ return false;
+ }
+
+ // Scheme is validated before the hostname because uri.host may not be
+ // populated for certain schemes. It appears to always be populated for
+ // https, so we avoid the potential NS_ERROR_FAILURE on field access.
+ if (uri.scheme != "https") {
+ this._log.debug("Source URI not HTTPS: " + uri.spec);
+ return false;
+ }
+
+ if (trustedHostnames.indexOf(uri.host) == -1) {
+ this._log.debug("Source hostname not trusted: " + uri.host);
+ return false;
+ }
+
+ return true;
+ },
+
+ /**
+ * Update the userDisabled flag on an add-on.
+ *
+ * This will enable or disable an add-on and call the supplied callback when
+ * the action is complete. If no action is needed, the callback gets called
+ * immediately.
+ *
+ * @param addon
+ * Addon instance to manipulate.
+ * @param value
+ * Boolean to which to set userDisabled on the passed Addon.
+ * @param callback
+ * Function to be called when action is complete. Will receive 2
+ * arguments, a truthy value that signifies error, and the Addon
+ * instance passed to this function.
+ */
+ updateUserDisabled: function updateUserDisabled(addon, value, callback) {
+ if (addon.userDisabled == value) {
+ callback(null, addon);
+ return;
+ }
+
+ // A pref allows changes to the enabled flag to be ignored.
+ if (Svc.Prefs.get("addons.ignoreUserEnabledChanges", false)) {
+ this._log.info("Ignoring enabled state change due to preference: " +
+ addon.id);
+ callback(null, addon);
+ return;
+ }
+
+ AddonUtils.updateUserDisabled(addon, value, callback);
+ },
+};
+
+/**
+ * The add-ons tracker keeps track of real-time changes to add-ons.
+ *
+ * It hooks up to the reconciler and receives notifications directly from it.
+ */
+function AddonsTracker(name, engine) {
+ Tracker.call(this, name, engine);
+}
+AddonsTracker.prototype = {
+ __proto__: Tracker.prototype,
+
+ get reconciler() {
+ return this.engine._reconciler;
+ },
+
+ get store() {
+ return this.engine._store;
+ },
+
+ /**
+ * This callback is executed whenever the AddonsReconciler sends out a change
+ * notification. See AddonsReconciler.addChangeListener().
+ */
+ changeListener: function changeHandler(date, change, addon) {
+ this._log.debug("changeListener invoked: " + change + " " + addon.id);
+ // Ignore changes that occur during sync.
+ if (this.ignoreAll) {
+ return;
+ }
+
+ if (!this.store.isAddonSyncable(addon)) {
+ this._log.debug("Ignoring change because add-on isn't syncable: " +
+ addon.id);
+ return;
+ }
+
+ this.addChangedID(addon.guid, date.getTime() / 1000);
+ this.score += SCORE_INCREMENT_XLARGE;
+ },
+
+ startTracking: function() {
+ if (this.engine.enabled) {
+ this.reconciler.startListening();
+ }
+
+ this.reconciler.addChangeListener(this);
+ },
+
+ stopTracking: function() {
+ this.reconciler.removeChangeListener(this);
+ this.reconciler.stopListening();
+ },
+};
+
+class AddonValidator extends CollectionValidator {
+ constructor(engine = null) {
+ super("addons", "id", [
+ "addonID",
+ "enabled",
+ "applicationID",
+ "source"
+ ]);
+ this.engine = engine;
+ }
+
+ getClientItems() {
+ return Promise.all([
+ new Promise(resolve =>
+ AddonManager.getAllAddons(resolve)),
+ new Promise(resolve =>
+ AddonManager.getAddonsWithOperationsByTypes(["extension", "theme"], resolve)),
+ ]).then(([installed, addonsWithPendingOperation]) => {
+ // Addons pending install won't be in the first list, but addons pending
+ // uninstall/enable/disable will be in both lists.
+ let all = new Map(installed.map(addon => [addon.id, addon]));
+ for (let addon of addonsWithPendingOperation) {
+ all.set(addon.id, addon);
+ }
+ // Convert to an array since Map.prototype.values returns an iterable
+ return [...all.values()];
+ });
+ }
+
+ normalizeClientItem(item) {
+ let enabled = !item.userDisabled;
+ if (item.pendingOperations & AddonManager.PENDING_ENABLE) {
+ enabled = true;
+ } else if (item.pendingOperations & AddonManager.PENDING_DISABLE) {
+ enabled = false;
+ }
+ return {
+ enabled,
+ id: item.syncGUID,
+ addonID: item.id,
+ applicationID: Services.appinfo.ID,
+ source: "amo", // check item.foreignInstall?
+ original: item
+ };
+ }
+
+ normalizeServerItem(item) {
+ let guid = this.engine._findDupe(item);
+ if (guid) {
+ item.id = guid;
+ }
+ return item;
+ }
+
+ clientUnderstands(item) {
+ return item.applicationID === Services.appinfo.ID;
+ }
+
+ syncedByClient(item) {
+ return !item.original.hidden &&
+ !item.original.isSystem &&
+ !(item.original.pendingOperations & AddonManager.PENDING_UNINSTALL) &&
+ this.engine.isAddonSyncable(item.original, true);
+ }
+}
diff --git a/services/sync/modules/engines/bookmarks.js b/services/sync/modules/engines/bookmarks.js
new file mode 100644
index 000000000..76a198a8b
--- /dev/null
+++ b/services/sync/modules/engines/bookmarks.js
@@ -0,0 +1,1378 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ['BookmarksEngine', "PlacesItem", "Bookmark",
+ "BookmarkFolder", "BookmarkQuery",
+ "Livemark", "BookmarkSeparator"];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cu = Components.utils;
+
+Cu.import("resource://gre/modules/PlacesUtils.jsm");
+Cu.import("resource://gre/modules/PlacesSyncUtils.jsm");
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+Cu.import("resource://services-common/async.js");
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://services-sync/engines.js");
+Cu.import("resource://services-sync/record.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://gre/modules/Task.jsm");
+Cu.import("resource://gre/modules/PlacesBackups.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "BookmarkValidator",
+ "resource://services-sync/bookmark_validator.js");
+XPCOMUtils.defineLazyGetter(this, "PlacesBundle", () => {
+ let bundleService = Cc["@mozilla.org/intl/stringbundle;1"]
+ .getService(Ci.nsIStringBundleService);
+ return bundleService.createBundle("chrome://places/locale/places.properties");
+});
+
+const ANNOS_TO_TRACK = [PlacesSyncUtils.bookmarks.DESCRIPTION_ANNO,
+ PlacesSyncUtils.bookmarks.SIDEBAR_ANNO,
+ PlacesUtils.LMANNO_FEEDURI, PlacesUtils.LMANNO_SITEURI];
+
+const SERVICE_NOT_SUPPORTED = "Service not supported on this platform";
+const FOLDER_SORTINDEX = 1000000;
+const {
+ SOURCE_SYNC,
+ SOURCE_IMPORT,
+ SOURCE_IMPORT_REPLACE,
+} = Ci.nsINavBookmarksService;
+
+const SQLITE_MAX_VARIABLE_NUMBER = 999;
+
+const ORGANIZERQUERY_ANNO = "PlacesOrganizer/OrganizerQuery";
+const ALLBOOKMARKS_ANNO = "AllBookmarks";
+const MOBILE_ANNO = "MobileBookmarks";
+
+// The tracker ignores changes made by bookmark import and restore, and
+// changes made by Sync. We don't need to exclude `SOURCE_IMPORT`, but both
+// import and restore fire `bookmarks-restore-*` observer notifications, and
+// the tracker doesn't currently distinguish between the two.
+const IGNORED_SOURCES = [SOURCE_SYNC, SOURCE_IMPORT, SOURCE_IMPORT_REPLACE];
+
+// Returns the constructor for a bookmark record type.
+function getTypeObject(type) {
+ switch (type) {
+ case "bookmark":
+ case "microsummary":
+ return Bookmark;
+ case "query":
+ return BookmarkQuery;
+ case "folder":
+ return BookmarkFolder;
+ case "livemark":
+ return Livemark;
+ case "separator":
+ return BookmarkSeparator;
+ case "item":
+ return PlacesItem;
+ }
+ return null;
+}
+
+this.PlacesItem = function PlacesItem(collection, id, type) {
+ CryptoWrapper.call(this, collection, id);
+ this.type = type || "item";
+}
+PlacesItem.prototype = {
+ decrypt: function PlacesItem_decrypt(keyBundle) {
+ // Do the normal CryptoWrapper decrypt, but change types before returning
+ let clear = CryptoWrapper.prototype.decrypt.call(this, keyBundle);
+
+ // Convert the abstract places item to the actual object type
+ if (!this.deleted)
+ this.__proto__ = this.getTypeObject(this.type).prototype;
+
+ return clear;
+ },
+
+ getTypeObject: function PlacesItem_getTypeObject(type) {
+ let recordObj = getTypeObject(type);
+ if (!recordObj) {
+ throw new Error("Unknown places item object type: " + type);
+ }
+ return recordObj;
+ },
+
+ __proto__: CryptoWrapper.prototype,
+ _logName: "Sync.Record.PlacesItem",
+
+ // Converts the record to a Sync bookmark object that can be passed to
+ // `PlacesSyncUtils.bookmarks.{insert, update}`.
+ toSyncBookmark() {
+ return {
+ kind: this.type,
+ syncId: this.id,
+ parentSyncId: this.parentid,
+ };
+ },
+
+ // Populates the record from a Sync bookmark object returned from
+ // `PlacesSyncUtils.bookmarks.fetch`.
+ fromSyncBookmark(item) {
+ this.parentid = item.parentSyncId;
+ this.parentName = item.parentTitle;
+ },
+};
+
+Utils.deferGetSet(PlacesItem,
+ "cleartext",
+ ["hasDupe", "parentid", "parentName", "type"]);
+
+this.Bookmark = function Bookmark(collection, id, type) {
+ PlacesItem.call(this, collection, id, type || "bookmark");
+}
+Bookmark.prototype = {
+ __proto__: PlacesItem.prototype,
+ _logName: "Sync.Record.Bookmark",
+
+ toSyncBookmark() {
+ let info = PlacesItem.prototype.toSyncBookmark.call(this);
+ info.title = this.title;
+ info.url = this.bmkUri;
+ info.description = this.description;
+ info.loadInSidebar = this.loadInSidebar;
+ info.tags = this.tags;
+ info.keyword = this.keyword;
+ return info;
+ },
+
+ fromSyncBookmark(item) {
+ PlacesItem.prototype.fromSyncBookmark.call(this, item);
+ this.title = item.title;
+ this.bmkUri = item.url.href;
+ this.description = item.description;
+ this.loadInSidebar = item.loadInSidebar;
+ this.tags = item.tags;
+ this.keyword = item.keyword;
+ },
+};
+
+Utils.deferGetSet(Bookmark,
+ "cleartext",
+ ["title", "bmkUri", "description",
+ "loadInSidebar", "tags", "keyword"]);
+
+this.BookmarkQuery = function BookmarkQuery(collection, id) {
+ Bookmark.call(this, collection, id, "query");
+}
+BookmarkQuery.prototype = {
+ __proto__: Bookmark.prototype,
+ _logName: "Sync.Record.BookmarkQuery",
+
+ toSyncBookmark() {
+ let info = Bookmark.prototype.toSyncBookmark.call(this);
+ info.folder = this.folderName;
+ info.query = this.queryId;
+ return info;
+ },
+
+ fromSyncBookmark(item) {
+ Bookmark.prototype.fromSyncBookmark.call(this, item);
+ this.folderName = item.folder;
+ this.queryId = item.query;
+ },
+};
+
+Utils.deferGetSet(BookmarkQuery,
+ "cleartext",
+ ["folderName", "queryId"]);
+
+this.BookmarkFolder = function BookmarkFolder(collection, id, type) {
+ PlacesItem.call(this, collection, id, type || "folder");
+}
+BookmarkFolder.prototype = {
+ __proto__: PlacesItem.prototype,
+ _logName: "Sync.Record.Folder",
+
+ toSyncBookmark() {
+ let info = PlacesItem.prototype.toSyncBookmark.call(this);
+ info.description = this.description;
+ info.title = this.title;
+ return info;
+ },
+
+ fromSyncBookmark(item) {
+ PlacesItem.prototype.fromSyncBookmark.call(this, item);
+ this.title = item.title;
+ this.description = item.description;
+ this.children = item.childSyncIds;
+ },
+};
+
+Utils.deferGetSet(BookmarkFolder, "cleartext", ["description", "title",
+ "children"]);
+
+this.Livemark = function Livemark(collection, id) {
+ BookmarkFolder.call(this, collection, id, "livemark");
+}
+Livemark.prototype = {
+ __proto__: BookmarkFolder.prototype,
+ _logName: "Sync.Record.Livemark",
+
+ toSyncBookmark() {
+ let info = BookmarkFolder.prototype.toSyncBookmark.call(this);
+ info.feed = this.feedUri;
+ info.site = this.siteUri;
+ return info;
+ },
+
+ fromSyncBookmark(item) {
+ BookmarkFolder.prototype.fromSyncBookmark.call(this, item);
+ this.feedUri = item.feed.href;
+ if (item.site) {
+ this.siteUri = item.site.href;
+ }
+ },
+};
+
+Utils.deferGetSet(Livemark, "cleartext", ["siteUri", "feedUri"]);
+
+this.BookmarkSeparator = function BookmarkSeparator(collection, id) {
+ PlacesItem.call(this, collection, id, "separator");
+}
+BookmarkSeparator.prototype = {
+ __proto__: PlacesItem.prototype,
+ _logName: "Sync.Record.Separator",
+
+ fromSyncBookmark(item) {
+ PlacesItem.prototype.fromSyncBookmark.call(this, item);
+ this.pos = item.index;
+ },
+};
+
+Utils.deferGetSet(BookmarkSeparator, "cleartext", "pos");
+
+this.BookmarksEngine = function BookmarksEngine(service) {
+ SyncEngine.call(this, "Bookmarks", service);
+}
+BookmarksEngine.prototype = {
+ __proto__: SyncEngine.prototype,
+ _recordObj: PlacesItem,
+ _storeObj: BookmarksStore,
+ _trackerObj: BookmarksTracker,
+ version: 2,
+ _defaultSort: "index",
+
+ syncPriority: 4,
+ allowSkippedRecord: false,
+
+ // A diagnostic helper to get the string value for a bookmark's URL given
+ // its ID. Always returns a string - on error will return a string in the
+ // form of "<description of error>" as this is purely for, eg, logging.
+ // (This means hitting the DB directly and we don't bother using a cached
+ // statement - we should rarely hit this.)
+ _getStringUrlForId(id) {
+ let url;
+ try {
+ let stmt = this._store._getStmt(`
+ SELECT h.url
+ FROM moz_places h
+ JOIN moz_bookmarks b ON h.id = b.fk
+ WHERE b.id = :id`);
+ stmt.params.id = id;
+ let rows = Async.querySpinningly(stmt, ["url"]);
+ url = rows.length == 0 ? "<not found>" : rows[0].url;
+ } catch (ex) {
+ if (Async.isShutdownException(ex)) {
+ throw ex;
+ }
+ if (ex instanceof Ci.mozIStorageError) {
+ url = `<failed: Storage error: ${ex.message} (${ex.result})>`;
+ } else {
+ url = `<failed: ${ex.toString()}>`;
+ }
+ }
+ return url;
+ },
+
+ _guidMapFailed: false,
+ _buildGUIDMap: function _buildGUIDMap() {
+ let store = this._store;
+ let guidMap = {};
+ let tree = Async.promiseSpinningly(PlacesUtils.promiseBookmarksTree("", {
+ includeItemIds: true
+ }));
+ function* walkBookmarksTree(tree, parent=null) {
+ if (tree) {
+ // Skip root node
+ if (parent) {
+ yield [tree, parent];
+ }
+ if (tree.children) {
+ for (let child of tree.children) {
+ store._sleep(0); // avoid jank while looping.
+ yield* walkBookmarksTree(child, tree);
+ }
+ }
+ }
+ }
+
+ function* walkBookmarksRoots(tree, rootIDs) {
+ for (let id of rootIDs) {
+ let bookmarkRoot = tree.children.find(child => child.id === id);
+ if (bookmarkRoot === null) {
+ continue;
+ }
+ yield* walkBookmarksTree(bookmarkRoot, tree);
+ }
+ }
+
+ let rootsToWalk = getChangeRootIds();
+
+ for (let [node, parent] of walkBookmarksRoots(tree, rootsToWalk)) {
+ let {guid, id, type: placeType} = node;
+ guid = PlacesSyncUtils.bookmarks.guidToSyncId(guid);
+ let key;
+ switch (placeType) {
+ case PlacesUtils.TYPE_X_MOZ_PLACE:
+ // Bookmark
+ let query = null;
+ if (node.annos && node.uri.startsWith("place:")) {
+ query = node.annos.find(({name}) =>
+ name === PlacesSyncUtils.bookmarks.SMART_BOOKMARKS_ANNO);
+ }
+ if (query && query.value) {
+ key = "q" + query.value;
+ } else {
+ key = "b" + node.uri + ":" + (node.title || "");
+ }
+ break;
+ case PlacesUtils.TYPE_X_MOZ_PLACE_CONTAINER:
+ // Folder
+ key = "f" + (node.title || "");
+ break;
+ case PlacesUtils.TYPE_X_MOZ_PLACE_SEPARATOR:
+ // Separator
+ key = "s" + node.index;
+ break;
+ default:
+ this._log.error("Unknown place type: '"+placeType+"'");
+ continue;
+ }
+
+ let parentName = parent.title || "";
+ if (guidMap[parentName] == null)
+ guidMap[parentName] = {};
+
+ // If the entry already exists, remember that there are explicit dupes.
+ let entry = new String(guid);
+ entry.hasDupe = guidMap[parentName][key] != null;
+
+ // Remember this item's GUID for its parent-name/key pair.
+ guidMap[parentName][key] = entry;
+ this._log.trace("Mapped: " + [parentName, key, entry, entry.hasDupe]);
+ }
+
+ return guidMap;
+ },
+
+ // Helper function to get a dupe GUID for an item.
+ _mapDupe: function _mapDupe(item) {
+ // Figure out if we have something to key with.
+ let key;
+ let altKey;
+ switch (item.type) {
+ case "query":
+ // Prior to Bug 610501, records didn't carry their Smart Bookmark
+ // anno, so we won't be able to dupe them correctly. This altKey
+ // hack should get them to dupe correctly.
+ if (item.queryId) {
+ key = "q" + item.queryId;
+ altKey = "b" + item.bmkUri + ":" + (item.title || "");
+ break;
+ }
+ // No queryID? Fall through to the regular bookmark case.
+ case "bookmark":
+ case "microsummary":
+ key = "b" + item.bmkUri + ":" + (item.title || "");
+ break;
+ case "folder":
+ case "livemark":
+ key = "f" + (item.title || "");
+ break;
+ case "separator":
+ key = "s" + item.pos;
+ break;
+ default:
+ return;
+ }
+
+ // Figure out if we have a map to use!
+ // This will throw in some circumstances. That's fine.
+ let guidMap = this._guidMap;
+
+ // Give the GUID if we have the matching pair.
+ let parentName = item.parentName || "";
+ this._log.trace("Finding mapping: " + parentName + ", " + key);
+ let parent = guidMap[parentName];
+
+ if (!parent) {
+ this._log.trace("No parent => no dupe.");
+ return undefined;
+ }
+
+ let dupe = parent[key];
+
+ if (dupe) {
+ this._log.trace("Mapped dupe: " + dupe);
+ return dupe;
+ }
+
+ if (altKey) {
+ dupe = parent[altKey];
+ if (dupe) {
+ this._log.trace("Mapped dupe using altKey " + altKey + ": " + dupe);
+ return dupe;
+ }
+ }
+
+ this._log.trace("No dupe found for key " + key + "/" + altKey + ".");
+ return undefined;
+ },
+
+ _syncStartup: function _syncStart() {
+ SyncEngine.prototype._syncStartup.call(this);
+
+ let cb = Async.makeSpinningCallback();
+ Task.spawn(function* () {
+ // For first-syncs, make a backup for the user to restore
+ if (this.lastSync == 0) {
+ this._log.debug("Bookmarks backup starting.");
+ yield PlacesBackups.create(null, true);
+ this._log.debug("Bookmarks backup done.");
+ }
+ }.bind(this)).then(
+ cb, ex => {
+ // Failure to create a backup is somewhat bad, but probably not bad
+ // enough to prevent syncing of bookmarks - so just log the error and
+ // continue.
+ this._log.warn("Error while backing up bookmarks, but continuing with sync", ex);
+ cb();
+ }
+ );
+
+ cb.wait();
+
+ this.__defineGetter__("_guidMap", function() {
+ // Create a mapping of folder titles and separator positions to GUID.
+ // We do this lazily so that we don't do any work unless we reconcile
+ // incoming items.
+ let guidMap;
+ try {
+ guidMap = this._buildGUIDMap();
+ } catch (ex) {
+ if (Async.isShutdownException(ex)) {
+ throw ex;
+ }
+ this._log.warn("Error while building GUID map, skipping all other incoming items", ex);
+ throw {code: Engine.prototype.eEngineAbortApplyIncoming,
+ cause: ex};
+ }
+ delete this._guidMap;
+ return this._guidMap = guidMap;
+ });
+
+ this._store._childrenToOrder = {};
+ this._store.clearPendingDeletions();
+ },
+
+ _deletePending() {
+ // Delete pending items -- See the comment above BookmarkStore's deletePending
+ let newlyModified = Async.promiseSpinningly(this._store.deletePending());
+ let now = this._tracker._now();
+ this._log.debug("Deleted pending items", newlyModified);
+ for (let modifiedSyncID of newlyModified) {
+ if (!this._modified.has(modifiedSyncID)) {
+ this._modified.set(modifiedSyncID, { timestamp: now, deleted: false });
+ }
+ }
+ },
+
+ // We avoid reviving folders since reviving them properly would require
+ // reviving their children as well. Unfortunately, this is the wrong choice
+ // in the case of a bookmark restore where wipeServer failed -- if the
+ // server has the folder as deleted, we *would* want to reupload this folder.
+ // This is mitigated by the fact that we move any undeleted children to the
+ // grandparent when deleting the parent.
+ _shouldReviveRemotelyDeletedRecord(item) {
+ let kind = Async.promiseSpinningly(
+ PlacesSyncUtils.bookmarks.getKindForSyncId(item.id));
+ if (kind === PlacesSyncUtils.bookmarks.KINDS.FOLDER) {
+ return false;
+ }
+
+ // In addition to preventing the deletion of this record (handled by the caller),
+ // we need to mark the parent of this record for uploading next sync, in order
+ // to ensure its children array is accurate.
+ let modifiedTimestamp = this._modified.getModifiedTimestamp(item.id);
+ if (!modifiedTimestamp) {
+ // We only expect this to be called with items locally modified, so
+ // something strange is going on - play it safe and don't revive it.
+ this._log.error("_shouldReviveRemotelyDeletedRecord called on unmodified item: " + item.id);
+ return false;
+ }
+
+ let localID = this._store.idForGUID(item.id);
+ let localParentID = PlacesUtils.bookmarks.getFolderIdForItem(localID);
+ let localParentSyncID = this._store.GUIDForId(localParentID);
+
+ this._log.trace(`Reviving item "${item.id}" and marking parent ${localParentSyncID} as modified.`);
+
+ if (!this._modified.has(localParentSyncID)) {
+ this._modified.set(localParentSyncID, {
+ timestamp: modifiedTimestamp,
+ deleted: false
+ });
+ }
+ return true
+ },
+
+ _processIncoming: function (newitems) {
+ try {
+ SyncEngine.prototype._processIncoming.call(this, newitems);
+ } finally {
+ try {
+ this._deletePending();
+ } finally {
+ // Reorder children.
+ this._store._orderChildren();
+ delete this._store._childrenToOrder;
+ }
+ }
+ },
+
+ _syncFinish: function _syncFinish() {
+ SyncEngine.prototype._syncFinish.call(this);
+ this._tracker._ensureMobileQuery();
+ },
+
+ _syncCleanup: function _syncCleanup() {
+ SyncEngine.prototype._syncCleanup.call(this);
+ delete this._guidMap;
+ },
+
+ _createRecord: function _createRecord(id) {
+ // Create the record as usual, but mark it as having dupes if necessary.
+ let record = SyncEngine.prototype._createRecord.call(this, id);
+ let entry = this._mapDupe(record);
+ if (entry != null && entry.hasDupe) {
+ record.hasDupe = true;
+ }
+ return record;
+ },
+
+ _findDupe: function _findDupe(item) {
+ this._log.trace("Finding dupe for " + item.id +
+ " (already duped: " + item.hasDupe + ").");
+
+ // Don't bother finding a dupe if the incoming item has duplicates.
+ if (item.hasDupe) {
+ this._log.trace(item.id + " already a dupe: not finding one.");
+ return;
+ }
+ let mapped = this._mapDupe(item);
+ this._log.debug(item.id + " mapped to " + mapped);
+ // We must return a string, not an object, and the entries in the GUIDMap
+ // are created via "new String()" making them an object.
+ return mapped ? mapped.toString() : mapped;
+ },
+
+ pullAllChanges() {
+ return new BookmarksChangeset(this._store.getAllIDs());
+ },
+
+ pullNewChanges() {
+ let modifiedGUIDs = this._getModifiedGUIDs();
+ if (!modifiedGUIDs.length) {
+ return new BookmarksChangeset(this._tracker.changedIDs);
+ }
+
+ // We don't use `PlacesUtils.promiseDBConnection` here because
+ // `getChangedIDs` might be called while we're in a batch, meaning we
+ // won't see any changes until the batch finishes and the transaction
+ // commits.
+ let db = PlacesUtils.history.QueryInterface(Ci.nsPIPlacesDatabase)
+ .DBConnection;
+
+ // Filter out tags, organizer queries, and other descendants that we're
+ // not tracking. We chunk `modifiedGUIDs` because SQLite limits the number
+ // of bound parameters per query.
+ for (let startIndex = 0;
+ startIndex < modifiedGUIDs.length;
+ startIndex += SQLITE_MAX_VARIABLE_NUMBER) {
+
+ let chunkLength = Math.min(SQLITE_MAX_VARIABLE_NUMBER,
+ modifiedGUIDs.length - startIndex);
+
+ let query = `
+ WITH RECURSIVE
+ modifiedGuids(guid) AS (
+ VALUES ${new Array(chunkLength).fill("(?)").join(", ")}
+ ),
+ syncedItems(id) AS (
+ VALUES ${getChangeRootIds().map(id => `(${id})`).join(", ")}
+ UNION ALL
+ SELECT b.id
+ FROM moz_bookmarks b
+ JOIN syncedItems s ON b.parent = s.id
+ )
+ SELECT b.guid
+ FROM modifiedGuids m
+ JOIN moz_bookmarks b ON b.guid = m.guid
+ LEFT JOIN syncedItems s ON b.id = s.id
+ WHERE s.id IS NULL
+ `;
+
+ let statement = db.createAsyncStatement(query);
+ try {
+ for (let i = 0; i < chunkLength; i++) {
+ statement.bindByIndex(i, modifiedGUIDs[startIndex + i]);
+ }
+ let results = Async.querySpinningly(statement, ["guid"]);
+ for (let { guid } of results) {
+ let syncID = PlacesSyncUtils.bookmarks.guidToSyncId(guid);
+ this._tracker.removeChangedID(syncID);
+ }
+ } finally {
+ statement.finalize();
+ }
+ }
+
+ return new BookmarksChangeset(this._tracker.changedIDs);
+ },
+
+ // Returns an array of Places GUIDs for all changed items. Ignores deletions,
+ // which won't exist in the DB and shouldn't be removed from the tracker.
+ _getModifiedGUIDs() {
+ let guids = [];
+ for (let syncID in this._tracker.changedIDs) {
+ if (this._tracker.changedIDs[syncID].deleted === true) {
+ // The `===` check also filters out old persisted timestamps,
+ // which won't have a `deleted` property.
+ continue;
+ }
+ let guid = PlacesSyncUtils.bookmarks.syncIdToGuid(syncID);
+ guids.push(guid);
+ }
+ return guids;
+ },
+
+ // Called when _findDupe returns a dupe item and the engine has decided to
+ // switch the existing item to the new incoming item.
+ _switchItemToDupe(localDupeGUID, incomingItem) {
+ // We unconditionally change the item's ID in case the engine knows of
+ // an item but doesn't expose it through itemExists. If the API
+ // contract were stronger, this could be changed.
+ this._log.debug("Switching local ID to incoming: " + localDupeGUID + " -> " +
+ incomingItem.id);
+ this._store.changeItemID(localDupeGUID, incomingItem.id);
+
+ // And mark the parent as being modified. Given we de-dupe based on the
+ // parent *name* it's possible the item having its GUID changed has a
+ // different parent from the incoming record.
+ // So we need to find the GUID of the local parent.
+ let now = this._tracker._now();
+ let localID = this._store.idForGUID(incomingItem.id);
+ let localParentID = PlacesUtils.bookmarks.getFolderIdForItem(localID);
+ let localParentGUID = this._store.GUIDForId(localParentID);
+ this._modified.set(localParentGUID, { modified: now, deleted: false });
+
+ // And we also add the parent as reflected in the incoming record as the
+ // de-dupe process might have used an existing item in a different folder.
+ // But only if the parent exists, otherwise we will upload a deleted item
+ // when it might actually be valid, just unknown to us. Note that this
+ // scenario will still leave us with inconsistent client and server states;
+ // the incoming record on the server references a parent that isn't the
+ // actual parent locally - see bug 1297955.
+ if (localParentGUID != incomingItem.parentid) {
+ let remoteParentID = this._store.idForGUID(incomingItem.parentid);
+ if (remoteParentID > 0) {
+ // The parent specified in the record does exist, so we are going to
+ // attempt a move when we come to applying the record. Mark the parent
+ // as being modified so we will later upload it with the new child
+ // reference.
+ this._modified.set(incomingItem.parentid, { modified: now, deleted: false });
+ } else {
+ // We aren't going to do a move as we don't have the parent (yet?).
+ // When applying the record we will add our special PARENT_ANNO
+ // annotation, so if it arrives in the future (either this Sync or a
+ // later one) it will be reparented.
+ this._log.debug(`Incoming duplicate item ${incomingItem.id} specifies ` +
+ `non-existing parent ${incomingItem.parentid}`);
+ }
+ }
+
+ // The local, duplicate ID is always deleted on the server - but for
+ // bookmarks it is a logical delete.
+ // Simply adding this (now non-existing) ID to the tracker is enough.
+ this._modified.set(localDupeGUID, { modified: now, deleted: true });
+ },
+ getValidator() {
+ return new BookmarkValidator();
+ }
+};
+
+function BookmarksStore(name, engine) {
+ Store.call(this, name, engine);
+ this._foldersToDelete = new Set();
+ this._atomsToDelete = new Set();
+ // Explicitly nullify our references to our cached services so we don't leak
+ Svc.Obs.add("places-shutdown", function() {
+ for (let query in this._stmts) {
+ let stmt = this._stmts[query];
+ stmt.finalize();
+ }
+ this._stmts = {};
+ }, this);
+}
+BookmarksStore.prototype = {
+ __proto__: Store.prototype,
+
+ itemExists: function BStore_itemExists(id) {
+ return this.idForGUID(id) > 0;
+ },
+
+ applyIncoming: function BStore_applyIncoming(record) {
+ this._log.debug("Applying record " + record.id);
+ let isSpecial = PlacesSyncUtils.bookmarks.ROOTS.includes(record.id);
+
+ if (record.deleted) {
+ if (isSpecial) {
+ this._log.warn("Ignoring deletion for special record " + record.id);
+ return;
+ }
+
+ // Don't bother with pre and post-processing for deletions.
+ Store.prototype.applyIncoming.call(this, record);
+ return;
+ }
+
+ // For special folders we're only interested in child ordering.
+ if (isSpecial && record.children) {
+ this._log.debug("Processing special node: " + record.id);
+ // Reorder children later
+ this._childrenToOrder[record.id] = record.children;
+ return;
+ }
+
+ // Skip malformed records. (Bug 806460.)
+ if (record.type == "query" &&
+ !record.bmkUri) {
+ this._log.warn("Skipping malformed query bookmark: " + record.id);
+ return;
+ }
+
+ // Figure out the local id of the parent GUID if available
+ let parentGUID = record.parentid;
+ if (!parentGUID) {
+ throw "Record " + record.id + " has invalid parentid: " + parentGUID;
+ }
+ this._log.debug("Remote parent is " + parentGUID);
+
+ // Do the normal processing of incoming records
+ Store.prototype.applyIncoming.call(this, record);
+
+ if (record.type == "folder" && record.children) {
+ this._childrenToOrder[record.id] = record.children;
+ }
+ },
+
+ create: function BStore_create(record) {
+ let info = record.toSyncBookmark();
+ // This can throw if we're inserting an invalid or incomplete bookmark.
+ // That's fine; the exception will be caught by `applyIncomingBatch`
+ // without aborting further processing.
+ let item = Async.promiseSpinningly(PlacesSyncUtils.bookmarks.insert(info));
+ if (item) {
+ this._log.debug(`Created ${item.kind} ${item.syncId} under ${
+ item.parentSyncId}`, item);
+ }
+ },
+
+ remove: function BStore_remove(record) {
+ if (PlacesSyncUtils.bookmarks.isRootSyncID(record.id)) {
+ this._log.warn("Refusing to remove special folder " + record.id);
+ return;
+ }
+ let recordKind = Async.promiseSpinningly(
+ PlacesSyncUtils.bookmarks.getKindForSyncId(record.id));
+ let isFolder = recordKind === PlacesSyncUtils.bookmarks.KINDS.FOLDER;
+ this._log.trace(`Buffering removal of item "${record.id}" of type "${recordKind}".`);
+ if (isFolder) {
+ this._foldersToDelete.add(record.id);
+ } else {
+ this._atomsToDelete.add(record.id);
+ }
+ },
+
+ update: function BStore_update(record) {
+ let info = record.toSyncBookmark();
+ let item = Async.promiseSpinningly(PlacesSyncUtils.bookmarks.update(info));
+ if (item) {
+ this._log.debug(`Updated ${item.kind} ${item.syncId} under ${
+ item.parentSyncId}`, item);
+ }
+ },
+
+ _orderChildren: function _orderChildren() {
+ let promises = Object.keys(this._childrenToOrder).map(syncID => {
+ let children = this._childrenToOrder[syncID];
+ return PlacesSyncUtils.bookmarks.order(syncID, children).catch(ex => {
+ this._log.debug(`Could not order children for ${syncID}`, ex);
+ });
+ });
+ Async.promiseSpinningly(Promise.all(promises));
+ },
+
+ // There's some complexity here around pending deletions. Our goals:
+ //
+ // - Don't delete any bookmarks a user has created but not explicitly deleted
+ // (This includes any bookmark that was not a child of the folder at the
+ // time the deletion was recorded, and also bookmarks restored from a backup).
+ // - Don't undelete any bookmark without ensuring the server structure
+ // includes it (see `BookmarkEngine.prototype._shouldReviveRemotelyDeletedRecord`)
+ //
+ // This leads the following approach:
+ //
+ // - Additions, moves, and updates are processed before deletions.
+ // - To do this, all deletion operations are buffered during a sync. Folders
+ // we plan on deleting have their sync id's stored in `this._foldersToDelete`,
+ // and non-folders we plan on deleting have their sync id's stored in
+ // `this._atomsToDelete`.
+ // - The exception to this is the moves that occur to fix the order of bookmark
+ // children, which are performed after we process deletions.
+ // - Non-folders are deleted before folder deletions, so that when we process
+ // folder deletions we know the correct state.
+ // - Remote deletions always win for folders, but do not result in recursive
+ // deletion of children. This is a hack because we're not able to distinguish
+ // between value changes and structural changes to folders, and we don't even
+ // have the old server record to compare to. See `BookmarkEngine`'s
+ // `_shouldReviveRemotelyDeletedRecord` method.
+ // - When a folder is deleted, its remaining children are moved in order to
+ // their closest living ancestor. If this is interrupted (unlikely, but
+ // possible given that we don't perform this operation in a transaction),
+ // we revive the folder.
+ // - Remote deletions can lose for non-folders, but only until we handle
+ // bookmark restores correctly (removing stale state from the server -- this
+ // is to say, if bug 1230011 is fixed, we should never revive bookmarks).
+
+ deletePending: Task.async(function* deletePending() {
+ yield this._deletePendingAtoms();
+ let guidsToUpdate = yield this._deletePendingFolders();
+ this.clearPendingDeletions();
+ return guidsToUpdate;
+ }),
+
+ clearPendingDeletions() {
+ this._foldersToDelete.clear();
+ this._atomsToDelete.clear();
+ },
+
+ _deleteAtom: Task.async(function* _deleteAtom(syncID) {
+ try {
+ let info = yield PlacesSyncUtils.bookmarks.remove(syncID, {
+ preventRemovalOfNonEmptyFolders: true
+ });
+ this._log.trace(`Removed item ${syncID} with type ${info.type}`);
+ } catch (ex) {
+ // Likely already removed.
+ this._log.trace(`Error removing ${syncID}`, ex);
+ }
+ }),
+
+ _deletePendingAtoms() {
+ return Promise.all(
+ [...this._atomsToDelete.values()]
+ .map(syncID => this._deleteAtom(syncID)));
+ },
+
+ // Returns an array of sync ids that need updates.
+ _deletePendingFolders: Task.async(function* _deletePendingFolders() {
+ // To avoid data loss, we don't want to just delete the folder outright,
+ // so we buffer folder deletions and process them at the end (now).
+ //
+ // At this point, any member in the folder that remains is either a folder
+ // pending deletion (which we'll get to in this function), or an item that
+ // should not be deleted. To avoid deleting these items, we first move them
+ // to the parent of the folder we're about to delete.
+ let needUpdate = new Set();
+ for (let syncId of this._foldersToDelete) {
+ let childSyncIds = yield PlacesSyncUtils.bookmarks.fetchChildSyncIds(syncId);
+ if (!childSyncIds.length) {
+ // No children -- just delete the folder.
+ yield this._deleteAtom(syncId)
+ continue;
+ }
+ // We could avoid some redundant work here by finding the nearest
+ // grandparent who isn't present in `this._toDelete`...
+
+ let grandparentSyncId = this.GUIDForId(
+ PlacesUtils.bookmarks.getFolderIdForItem(
+ this.idForGUID(PlacesSyncUtils.bookmarks.syncIdToGuid(syncId))));
+
+ this._log.trace(`Moving ${childSyncIds.length} children of "${syncId}" to ` +
+ `grandparent "${grandparentSyncId}" before deletion.`);
+
+ // Move children out of the parent and into the grandparent
+ yield Promise.all(childSyncIds.map(child => PlacesSyncUtils.bookmarks.update({
+ syncId: child,
+ parentSyncId: grandparentSyncId
+ })));
+
+ // Delete the (now empty) parent
+ try {
+ yield PlacesSyncUtils.bookmarks.remove(syncId, {
+ preventRemovalOfNonEmptyFolders: true
+ });
+ } catch (e) {
+ // We failed, probably because someone added something to this folder
+ // between when we got the children and now (or the database is corrupt,
+ // or something else happened...) This is unlikely, but possible. To
+ // avoid corruption in this case, we need to reupload the record to the
+ // server.
+ //
+ // (Ideally this whole operation would be done in a transaction, and this
+ // wouldn't be possible).
+ needUpdate.add(syncId);
+ }
+
+ // Add children (for parentid) and grandparent (for children list) to set
+ // of records needing an update, *unless* they're marked for deletion.
+ if (!this._foldersToDelete.has(grandparentSyncId)) {
+ needUpdate.add(grandparentSyncId);
+ }
+ for (let childSyncId of childSyncIds) {
+ if (!this._foldersToDelete.has(childSyncId)) {
+ needUpdate.add(childSyncId);
+ }
+ }
+ }
+ return [...needUpdate];
+ }),
+
+ changeItemID: function BStore_changeItemID(oldID, newID) {
+ this._log.debug("Changing GUID " + oldID + " to " + newID);
+
+ Async.promiseSpinningly(PlacesSyncUtils.bookmarks.changeGuid(oldID, newID));
+ },
+
+ // Create a record starting from the weave id (places guid)
+ createRecord: function createRecord(id, collection) {
+ let item = Async.promiseSpinningly(PlacesSyncUtils.bookmarks.fetch(id));
+ if (!item) { // deleted item
+ let record = new PlacesItem(collection, id);
+ record.deleted = true;
+ return record;
+ }
+
+ let recordObj = getTypeObject(item.kind);
+ if (!recordObj) {
+ this._log.warn("Unknown item type, cannot serialize: " + item.kind);
+ recordObj = PlacesItem;
+ }
+ let record = new recordObj(collection, id);
+ record.fromSyncBookmark(item);
+
+ record.sortindex = this._calculateIndex(record);
+
+ return record;
+ },
+
+ _stmts: {},
+ _getStmt: function(query) {
+ if (query in this._stmts) {
+ return this._stmts[query];
+ }
+
+ this._log.trace("Creating SQL statement: " + query);
+ let db = PlacesUtils.history.QueryInterface(Ci.nsPIPlacesDatabase)
+ .DBConnection;
+ return this._stmts[query] = db.createAsyncStatement(query);
+ },
+
+ get _frecencyStm() {
+ return this._getStmt(
+ "SELECT frecency " +
+ "FROM moz_places " +
+ "WHERE url_hash = hash(:url) AND url = :url " +
+ "LIMIT 1");
+ },
+ _frecencyCols: ["frecency"],
+
+ GUIDForId: function GUIDForId(id) {
+ let guid = Async.promiseSpinningly(PlacesUtils.promiseItemGuid(id));
+ return PlacesSyncUtils.bookmarks.guidToSyncId(guid);
+ },
+
+ idForGUID: function idForGUID(guid) {
+ // guid might be a String object rather than a string.
+ guid = PlacesSyncUtils.bookmarks.syncIdToGuid(guid.toString());
+
+ return Async.promiseSpinningly(PlacesUtils.promiseItemId(guid).catch(
+ ex => -1));
+ },
+
+ _calculateIndex: function _calculateIndex(record) {
+ // Ensure folders have a very high sort index so they're not synced last.
+ if (record.type == "folder")
+ return FOLDER_SORTINDEX;
+
+ // For anything directly under the toolbar, give it a boost of more than an
+ // unvisited bookmark
+ let index = 0;
+ if (record.parentid == "toolbar")
+ index += 150;
+
+ // Add in the bookmark's frecency if we have something.
+ if (record.bmkUri != null) {
+ this._frecencyStm.params.url = record.bmkUri;
+ let result = Async.querySpinningly(this._frecencyStm, this._frecencyCols);
+ if (result.length)
+ index += result[0].frecency;
+ }
+
+ return index;
+ },
+
+ getAllIDs: function BStore_getAllIDs() {
+ let items = {};
+
+ let query = `
+ WITH RECURSIVE
+ changeRootContents(id) AS (
+ VALUES ${getChangeRootIds().map(id => `(${id})`).join(", ")}
+ UNION ALL
+ SELECT b.id
+ FROM moz_bookmarks b
+ JOIN changeRootContents c ON b.parent = c.id
+ )
+ SELECT guid
+ FROM changeRootContents
+ JOIN moz_bookmarks USING (id)
+ `;
+
+ let statement = this._getStmt(query);
+ let results = Async.querySpinningly(statement, ["guid"]);
+ for (let { guid } of results) {
+ let syncID = PlacesSyncUtils.bookmarks.guidToSyncId(guid);
+ items[syncID] = { modified: 0, deleted: false };
+ }
+
+ return items;
+ },
+
+ wipe: function BStore_wipe() {
+ this.clearPendingDeletions();
+ Async.promiseSpinningly(Task.spawn(function* () {
+ // Save a backup before clearing out all bookmarks.
+ yield PlacesBackups.create(null, true);
+ yield PlacesUtils.bookmarks.eraseEverything({
+ source: SOURCE_SYNC,
+ });
+ }));
+ }
+};
+
+function BookmarksTracker(name, engine) {
+ this._batchDepth = 0;
+ this._batchSawScoreIncrement = false;
+ Tracker.call(this, name, engine);
+
+ Svc.Obs.add("places-shutdown", this);
+}
+BookmarksTracker.prototype = {
+ __proto__: Tracker.prototype,
+
+ //`_ignore` checks the change source for each observer notification, so we
+ // don't want to let the engine ignore all changes during a sync.
+ get ignoreAll() {
+ return false;
+ },
+
+ // Define an empty setter so that the engine doesn't throw a `TypeError`
+ // setting a read-only property.
+ set ignoreAll(value) {},
+
+ startTracking: function() {
+ PlacesUtils.bookmarks.addObserver(this, true);
+ Svc.Obs.add("bookmarks-restore-begin", this);
+ Svc.Obs.add("bookmarks-restore-success", this);
+ Svc.Obs.add("bookmarks-restore-failed", this);
+ },
+
+ stopTracking: function() {
+ PlacesUtils.bookmarks.removeObserver(this);
+ Svc.Obs.remove("bookmarks-restore-begin", this);
+ Svc.Obs.remove("bookmarks-restore-success", this);
+ Svc.Obs.remove("bookmarks-restore-failed", this);
+ },
+
+ observe: function observe(subject, topic, data) {
+ Tracker.prototype.observe.call(this, subject, topic, data);
+
+ switch (topic) {
+ case "bookmarks-restore-begin":
+ this._log.debug("Ignoring changes from importing bookmarks.");
+ break;
+ case "bookmarks-restore-success":
+ this._log.debug("Tracking all items on successful import.");
+
+ this._log.debug("Restore succeeded: wiping server and other clients.");
+ this.engine.service.resetClient([this.name]);
+ this.engine.service.wipeServer([this.name]);
+ this.engine.service.clientsEngine.sendCommand("wipeEngine", [this.name]);
+ break;
+ case "bookmarks-restore-failed":
+ this._log.debug("Tracking all items on failed import.");
+ break;
+ }
+ },
+
+ QueryInterface: XPCOMUtils.generateQI([
+ Ci.nsINavBookmarkObserver,
+ Ci.nsINavBookmarkObserver_MOZILLA_1_9_1_ADDITIONS,
+ Ci.nsISupportsWeakReference
+ ]),
+
+ addChangedID(id, change) {
+ if (!id) {
+ this._log.warn("Attempted to add undefined ID to tracker");
+ return false;
+ }
+ if (this._ignored.includes(id)) {
+ return false;
+ }
+ let shouldSaveChange = false;
+ let currentChange = this.changedIDs[id];
+ if (currentChange) {
+ if (typeof currentChange == "number") {
+ // Allow raw timestamps for backward-compatibility with persisted
+ // changed IDs. The new format uses tuples to track deleted items.
+ shouldSaveChange = currentChange < change.modified;
+ } else {
+ shouldSaveChange = currentChange.modified < change.modified ||
+ currentChange.deleted != change.deleted;
+ }
+ } else {
+ shouldSaveChange = true;
+ }
+ if (shouldSaveChange) {
+ this._saveChangedID(id, change);
+ }
+ return true;
+ },
+
+ /**
+ * Add a bookmark GUID to be uploaded and bump up the sync score.
+ *
+ * @param itemId
+ * The Places item ID of the bookmark to upload.
+ * @param guid
+ * The Places GUID of the bookmark to upload.
+ * @param isTombstone
+ * Whether we're uploading a tombstone for a removed bookmark.
+ */
+ _add: function BMT__add(itemId, guid, isTombstone = false) {
+ let syncID = PlacesSyncUtils.bookmarks.guidToSyncId(guid);
+ let info = { modified: Date.now() / 1000, deleted: isTombstone };
+ if (this.addChangedID(syncID, info)) {
+ this._upScore();
+ }
+ },
+
+ /* Every add/remove/change will trigger a sync for MULTI_DEVICE (except in
+ a batch operation, where we do it at the end of the batch) */
+ _upScore: function BMT__upScore() {
+ if (this._batchDepth == 0) {
+ this.score += SCORE_INCREMENT_XLARGE;
+ } else {
+ this._batchSawScoreIncrement = true;
+ }
+ },
+
+ onItemAdded: function BMT_onItemAdded(itemId, folder, index,
+ itemType, uri, title, dateAdded,
+ guid, parentGuid, source) {
+ if (IGNORED_SOURCES.includes(source)) {
+ return;
+ }
+
+ this._log.trace("onItemAdded: " + itemId);
+ this._add(itemId, guid);
+ this._add(folder, parentGuid);
+ },
+
+ onItemRemoved: function (itemId, parentId, index, type, uri,
+ guid, parentGuid, source) {
+ if (IGNORED_SOURCES.includes(source)) {
+ return;
+ }
+
+ // Ignore changes to tags (folders under the tags folder).
+ if (parentId == PlacesUtils.tagsFolderId) {
+ return;
+ }
+
+ let grandParentId = -1;
+ try {
+ grandParentId = PlacesUtils.bookmarks.getFolderIdForItem(parentId);
+ } catch (ex) {
+ // `getFolderIdForItem` can throw if the item no longer exists, such as
+ // when we've removed a subtree using `removeFolderChildren`.
+ return;
+ }
+
+ // Ignore tag items (the actual instance of a tag for a bookmark).
+ if (grandParentId == PlacesUtils.tagsFolderId) {
+ return;
+ }
+
+ /**
+ * The above checks are incomplete: we can still write tombstones for
+ * items that we don't track, and upload extraneous roots.
+ *
+ * Consider the left pane root: it's a child of the Places root, and has
+ * children and grandchildren. `PlacesUIUtils` can create, delete, and
+ * recreate it as needed. We can't determine ancestors when the root or its
+ * children are deleted, because they've already been removed from the
+ * database when `onItemRemoved` is called. Likewise, we can't check their
+ * "exclude from backup" annos, because they've *also* been removed.
+ *
+ * So, we end up writing tombstones for the left pane queries and left
+ * pane root. For good measure, we'll also upload the Places root, because
+ * it's the parent of the left pane root.
+ *
+ * As a workaround, we can track the parent GUID and reconstruct the item's
+ * ancestry at sync time. This is complicated, and the previous behavior was
+ * already wrong, so we'll wait for bug 1258127 to fix this generally.
+ */
+ this._log.trace("onItemRemoved: " + itemId);
+ this._add(itemId, guid, /* isTombstone */ true);
+ this._add(parentId, parentGuid);
+ },
+
+ _ensureMobileQuery: function _ensureMobileQuery() {
+ let find = val =>
+ PlacesUtils.annotations.getItemsWithAnnotation(ORGANIZERQUERY_ANNO, {}).filter(
+ id => PlacesUtils.annotations.getItemAnnotation(id, ORGANIZERQUERY_ANNO) == val
+ );
+
+ // Don't continue if the Library isn't ready
+ let all = find(ALLBOOKMARKS_ANNO);
+ if (all.length == 0)
+ return;
+
+ let mobile = find(MOBILE_ANNO);
+ let queryURI = Utils.makeURI("place:folder=" + PlacesUtils.mobileFolderId);
+ let title = PlacesBundle.GetStringFromName("MobileBookmarksFolderTitle");
+
+ // Don't add OR remove the mobile bookmarks if there's nothing.
+ if (PlacesUtils.bookmarks.getIdForItemAt(PlacesUtils.mobileFolderId, 0) == -1) {
+ if (mobile.length != 0)
+ PlacesUtils.bookmarks.removeItem(mobile[0], SOURCE_SYNC);
+ }
+ // Add the mobile bookmarks query if it doesn't exist
+ else if (mobile.length == 0) {
+ let query = PlacesUtils.bookmarks.insertBookmark(all[0], queryURI, -1, title, /* guid */ null, SOURCE_SYNC);
+ PlacesUtils.annotations.setItemAnnotation(query, ORGANIZERQUERY_ANNO, MOBILE_ANNO, 0,
+ PlacesUtils.annotations.EXPIRE_NEVER, SOURCE_SYNC);
+ PlacesUtils.annotations.setItemAnnotation(query, PlacesUtils.EXCLUDE_FROM_BACKUP_ANNO, 1, 0,
+ PlacesUtils.annotations.EXPIRE_NEVER, SOURCE_SYNC);
+ }
+ // Make sure the existing query URL and title are correct
+ else {
+ if (!PlacesUtils.bookmarks.getBookmarkURI(mobile[0]).equals(queryURI)) {
+ PlacesUtils.bookmarks.changeBookmarkURI(mobile[0], queryURI,
+ SOURCE_SYNC);
+ }
+ let queryTitle = PlacesUtils.bookmarks.getItemTitle(mobile[0]);
+ if (queryTitle != title) {
+ PlacesUtils.bookmarks.setItemTitle(mobile[0], title, SOURCE_SYNC);
+ }
+ let rootTitle =
+ PlacesUtils.bookmarks.getItemTitle(PlacesUtils.mobileFolderId);
+ if (rootTitle != title) {
+ PlacesUtils.bookmarks.setItemTitle(PlacesUtils.mobileFolderId, title,
+ SOURCE_SYNC);
+ }
+ }
+ },
+
+ // This method is oddly structured, but the idea is to return as quickly as
+ // possible -- this handler gets called *every time* a bookmark changes, for
+ // *each change*.
+ onItemChanged: function BMT_onItemChanged(itemId, property, isAnno, value,
+ lastModified, itemType, parentId,
+ guid, parentGuid, oldValue,
+ source) {
+ if (IGNORED_SOURCES.includes(source)) {
+ return;
+ }
+
+ if (isAnno && (ANNOS_TO_TRACK.indexOf(property) == -1))
+ // Ignore annotations except for the ones that we sync.
+ return;
+
+ // Ignore favicon changes to avoid unnecessary churn.
+ if (property == "favicon")
+ return;
+
+ this._log.trace("onItemChanged: " + itemId +
+ (", " + property + (isAnno? " (anno)" : "")) +
+ (value ? (" = \"" + value + "\"") : ""));
+ this._add(itemId, guid);
+ },
+
+ onItemMoved: function BMT_onItemMoved(itemId, oldParent, oldIndex,
+ newParent, newIndex, itemType,
+ guid, oldParentGuid, newParentGuid,
+ source) {
+ if (IGNORED_SOURCES.includes(source)) {
+ return;
+ }
+
+ this._log.trace("onItemMoved: " + itemId);
+ this._add(oldParent, oldParentGuid);
+ if (oldParent != newParent) {
+ this._add(itemId, guid);
+ this._add(newParent, newParentGuid);
+ }
+
+ // Remove any position annotations now that the user moved the item
+ PlacesUtils.annotations.removeItemAnnotation(itemId,
+ PlacesSyncUtils.bookmarks.SYNC_PARENT_ANNO, SOURCE_SYNC);
+ },
+
+ onBeginUpdateBatch: function () {
+ ++this._batchDepth;
+ },
+ onEndUpdateBatch: function () {
+ if (--this._batchDepth === 0 && this._batchSawScoreIncrement) {
+ this.score += SCORE_INCREMENT_XLARGE;
+ this._batchSawScoreIncrement = false;
+ }
+ },
+ onItemVisited: function () {}
+};
+
+// Returns an array of root IDs to recursively query for synced bookmarks.
+// Items in other roots, including tags and organizer queries, will be
+// ignored.
+function getChangeRootIds() {
+ return [
+ PlacesUtils.bookmarksMenuFolderId,
+ PlacesUtils.toolbarFolderId,
+ PlacesUtils.unfiledBookmarksFolderId,
+ PlacesUtils.mobileFolderId,
+ ];
+}
+
+class BookmarksChangeset extends Changeset {
+ getModifiedTimestamp(id) {
+ let change = this.changes[id];
+ return change ? change.modified : Number.NaN;
+ }
+}
diff --git a/services/sync/modules/engines/clients.js b/services/sync/modules/engines/clients.js
new file mode 100644
index 000000000..3dd679570
--- /dev/null
+++ b/services/sync/modules/engines/clients.js
@@ -0,0 +1,782 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * How does the clients engine work?
+ *
+ * - We use 2 files - commands.json and commands-syncing.json.
+ *
+ * - At sync upload time, we attempt a rename of commands.json to
+ * commands-syncing.json, and ignore errors (helps for crash during sync!).
+ * - We load commands-syncing.json and stash the contents in
+ * _currentlySyncingCommands which lives for the duration of the upload process.
+ * - We use _currentlySyncingCommands to build the outgoing records
+ * - Immediately after successful upload, we delete commands-syncing.json from
+ * disk (and clear _currentlySyncingCommands). We reconcile our local records
+ * with what we just wrote in the server, and add failed IDs commands
+ * back in commands.json
+ * - Any time we need to "save" a command for future syncs, we load
+ * commands.json, update it, and write it back out.
+ */
+
+this.EXPORTED_SYMBOLS = [
+ "ClientEngine",
+ "ClientsRec"
+];
+
+var {classes: Cc, interfaces: Ci, utils: Cu} = Components;
+
+Cu.import("resource://services-common/async.js");
+Cu.import("resource://services-common/stringbundle.js");
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://services-sync/engines.js");
+Cu.import("resource://services-sync/record.js");
+Cu.import("resource://services-sync/resource.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://gre/modules/Services.jsm");
+
+XPCOMUtils.defineLazyModuleGetter(this, "fxAccounts",
+ "resource://gre/modules/FxAccounts.jsm");
+
+const CLIENTS_TTL = 1814400; // 21 days
+const CLIENTS_TTL_REFRESH = 604800; // 7 days
+const STALE_CLIENT_REMOTE_AGE = 604800; // 7 days
+
+const SUPPORTED_PROTOCOL_VERSIONS = ["1.1", "1.5"];
+
+function hasDupeCommand(commands, action) {
+ if (!commands) {
+ return false;
+ }
+ return commands.some(other => other.command == action.command &&
+ Utils.deepEquals(other.args, action.args));
+}
+
+this.ClientsRec = function ClientsRec(collection, id) {
+ CryptoWrapper.call(this, collection, id);
+}
+ClientsRec.prototype = {
+ __proto__: CryptoWrapper.prototype,
+ _logName: "Sync.Record.Clients",
+ ttl: CLIENTS_TTL
+};
+
+Utils.deferGetSet(ClientsRec,
+ "cleartext",
+ ["name", "type", "commands",
+ "version", "protocols",
+ "formfactor", "os", "appPackage", "application", "device",
+ "fxaDeviceId"]);
+
+
+this.ClientEngine = function ClientEngine(service) {
+ SyncEngine.call(this, "Clients", service);
+
+ // Reset the last sync timestamp on every startup so that we fetch all clients
+ this.resetLastSync();
+}
+ClientEngine.prototype = {
+ __proto__: SyncEngine.prototype,
+ _storeObj: ClientStore,
+ _recordObj: ClientsRec,
+ _trackerObj: ClientsTracker,
+ allowSkippedRecord: false,
+
+ // Always sync client data as it controls other sync behavior
+ get enabled() {
+ return true;
+ },
+
+ get lastRecordUpload() {
+ return Svc.Prefs.get(this.name + ".lastRecordUpload", 0);
+ },
+ set lastRecordUpload(value) {
+ Svc.Prefs.set(this.name + ".lastRecordUpload", Math.floor(value));
+ },
+
+ get remoteClients() {
+ // return all non-stale clients for external consumption.
+ return Object.values(this._store._remoteClients).filter(v => !v.stale);
+ },
+
+ remoteClientExists(id) {
+ let client = this._store._remoteClients[id];
+ return !!(client && !client.stale);
+ },
+
+ // Aggregate some stats on the composition of clients on this account
+ get stats() {
+ let stats = {
+ hasMobile: this.localType == DEVICE_TYPE_MOBILE,
+ names: [this.localName],
+ numClients: 1,
+ };
+
+ for (let id in this._store._remoteClients) {
+ let {name, type, stale} = this._store._remoteClients[id];
+ if (!stale) {
+ stats.hasMobile = stats.hasMobile || type == DEVICE_TYPE_MOBILE;
+ stats.names.push(name);
+ stats.numClients++;
+ }
+ }
+
+ return stats;
+ },
+
+ /**
+ * Obtain information about device types.
+ *
+ * Returns a Map of device types to integer counts.
+ */
+ get deviceTypes() {
+ let counts = new Map();
+
+ counts.set(this.localType, 1);
+
+ for (let id in this._store._remoteClients) {
+ let record = this._store._remoteClients[id];
+ if (record.stale) {
+ continue; // pretend "stale" records don't exist.
+ }
+ let type = record.type;
+ if (!counts.has(type)) {
+ counts.set(type, 0);
+ }
+
+ counts.set(type, counts.get(type) + 1);
+ }
+
+ return counts;
+ },
+
+ get localID() {
+ // Generate a random GUID id we don't have one
+ let localID = Svc.Prefs.get("client.GUID", "");
+ return localID == "" ? this.localID = Utils.makeGUID() : localID;
+ },
+ set localID(value) {
+ Svc.Prefs.set("client.GUID", value);
+ },
+
+ get brandName() {
+ let brand = new StringBundle("chrome://branding/locale/brand.properties");
+ return brand.get("brandShortName");
+ },
+
+ get localName() {
+ let name = Utils.getDeviceName();
+ // If `getDeviceName` returns the default name, set the pref. FxA registers
+ // the device before syncing, so we don't need to update the registration
+ // in this case.
+ Svc.Prefs.set("client.name", name);
+ return name;
+ },
+ set localName(value) {
+ Svc.Prefs.set("client.name", value);
+ // Update the registration in the background.
+ fxAccounts.updateDeviceRegistration().catch(error => {
+ this._log.warn("failed to update fxa device registration", error);
+ });
+ },
+
+ get localType() {
+ return Utils.getDeviceType();
+ },
+ set localType(value) {
+ Svc.Prefs.set("client.type", value);
+ },
+
+ getClientName(id) {
+ if (id == this.localID) {
+ return this.localName;
+ }
+ let client = this._store._remoteClients[id];
+ return client ? client.name : "";
+ },
+
+ getClientFxaDeviceId(id) {
+ if (this._store._remoteClients[id]) {
+ return this._store._remoteClients[id].fxaDeviceId;
+ }
+ return null;
+ },
+
+ isMobile: function isMobile(id) {
+ if (this._store._remoteClients[id])
+ return this._store._remoteClients[id].type == DEVICE_TYPE_MOBILE;
+ return false;
+ },
+
+ _readCommands() {
+ let cb = Async.makeSpinningCallback();
+ Utils.jsonLoad("commands", this, commands => cb(null, commands));
+ return cb.wait() || {};
+ },
+
+ /**
+ * Low level function, do not use directly (use _addClientCommand instead).
+ */
+ _saveCommands(commands) {
+ let cb = Async.makeSpinningCallback();
+ Utils.jsonSave("commands", this, commands, error => {
+ if (error) {
+ this._log.error("Failed to save JSON outgoing commands", error);
+ }
+ cb();
+ });
+ cb.wait();
+ },
+
+ _prepareCommandsForUpload() {
+ let cb = Async.makeSpinningCallback();
+ Utils.jsonMove("commands", "commands-syncing", this).catch(() => {}) // Ignore errors
+ .then(() => {
+ Utils.jsonLoad("commands-syncing", this, commands => cb(null, commands));
+ });
+ return cb.wait() || {};
+ },
+
+ _deleteUploadedCommands() {
+ delete this._currentlySyncingCommands;
+ Async.promiseSpinningly(
+ Utils.jsonRemove("commands-syncing", this).catch(err => {
+ this._log.error("Failed to delete syncing-commands file", err);
+ })
+ );
+ },
+
+ _addClientCommand(clientId, command) {
+ const allCommands = this._readCommands();
+ const clientCommands = allCommands[clientId] || [];
+ if (hasDupeCommand(clientCommands, command)) {
+ return;
+ }
+ allCommands[clientId] = clientCommands.concat(command);
+ this._saveCommands(allCommands);
+ },
+
+ _syncStartup: function _syncStartup() {
+ // Reupload new client record periodically.
+ if (Date.now() / 1000 - this.lastRecordUpload > CLIENTS_TTL_REFRESH) {
+ this._tracker.addChangedID(this.localID);
+ this.lastRecordUpload = Date.now() / 1000;
+ }
+ SyncEngine.prototype._syncStartup.call(this);
+ },
+
+ _processIncoming() {
+ // Fetch all records from the server.
+ this.lastSync = 0;
+ this._incomingClients = {};
+ try {
+ SyncEngine.prototype._processIncoming.call(this);
+ // Since clients are synced unconditionally, any records in the local store
+ // that don't exist on the server must be for disconnected clients. Remove
+ // them, so that we don't upload records with commands for clients that will
+ // never see them. We also do this to filter out stale clients from the
+ // tabs collection, since showing their list of tabs is confusing.
+ for (let id in this._store._remoteClients) {
+ if (!this._incomingClients[id]) {
+ this._log.info(`Removing local state for deleted client ${id}`);
+ this._removeRemoteClient(id);
+ }
+ }
+ // Bug 1264498: Mobile clients don't remove themselves from the clients
+ // collection when the user disconnects Sync, so we mark as stale clients
+ // with the same name that haven't synced in over a week.
+ // (Note we can't simply delete them, or we re-apply them next sync - see
+ // bug 1287687)
+ delete this._incomingClients[this.localID];
+ let names = new Set([this.localName]);
+ for (let id in this._incomingClients) {
+ let record = this._store._remoteClients[id];
+ if (!names.has(record.name)) {
+ names.add(record.name);
+ continue;
+ }
+ let remoteAge = AsyncResource.serverTime - this._incomingClients[id];
+ if (remoteAge > STALE_CLIENT_REMOTE_AGE) {
+ this._log.info(`Hiding stale client ${id} with age ${remoteAge}`);
+ record.stale = true;
+ }
+ }
+ } finally {
+ this._incomingClients = null;
+ }
+ },
+
+ _uploadOutgoing() {
+ this._currentlySyncingCommands = this._prepareCommandsForUpload();
+ const clientWithPendingCommands = Object.keys(this._currentlySyncingCommands);
+ for (let clientId of clientWithPendingCommands) {
+ if (this._store._remoteClients[clientId] || this.localID == clientId) {
+ this._modified.set(clientId, 0);
+ }
+ }
+ SyncEngine.prototype._uploadOutgoing.call(this);
+ },
+
+ _onRecordsWritten(succeeded, failed) {
+ // Reconcile the status of the local records with what we just wrote on the
+ // server
+ for (let id of succeeded) {
+ const commandChanges = this._currentlySyncingCommands[id];
+ if (id == this.localID) {
+ if (this.localCommands) {
+ this.localCommands = this.localCommands.filter(command => !hasDupeCommand(commandChanges, command));
+ }
+ } else {
+ const clientRecord = this._store._remoteClients[id];
+ if (!commandChanges || !clientRecord) {
+ // should be impossible, else we wouldn't have been writing it.
+ this._log.warn("No command/No record changes for a client we uploaded");
+ continue;
+ }
+ // fixup the client record, so our copy of _remoteClients matches what we uploaded.
+ clientRecord.commands = this._store.createRecord(id);
+ // we could do better and pass the reference to the record we just uploaded,
+ // but this will do for now
+ }
+ }
+
+ // Re-add failed commands
+ for (let id of failed) {
+ const commandChanges = this._currentlySyncingCommands[id];
+ if (!commandChanges) {
+ continue;
+ }
+ this._addClientCommand(id, commandChanges);
+ }
+
+ this._deleteUploadedCommands();
+
+ // Notify other devices that their own client collection changed
+ const idsToNotify = succeeded.reduce((acc, id) => {
+ if (id == this.localID) {
+ return acc;
+ }
+ const fxaDeviceId = this.getClientFxaDeviceId(id);
+ return fxaDeviceId ? acc.concat(fxaDeviceId) : acc;
+ }, []);
+ if (idsToNotify.length > 0) {
+ this._notifyCollectionChanged(idsToNotify);
+ }
+ },
+
+ _notifyCollectionChanged(ids) {
+ const message = {
+ version: 1,
+ command: "sync:collection_changed",
+ data: {
+ collections: ["clients"]
+ }
+ };
+ fxAccounts.notifyDevices(ids, message, NOTIFY_TAB_SENT_TTL_SECS);
+ },
+
+ _syncFinish() {
+ // Record histograms for our device types, and also write them to a pref
+ // so non-histogram telemetry (eg, UITelemetry) has easy access to them.
+ for (let [deviceType, count] of this.deviceTypes) {
+ let hid;
+ let prefName = this.name + ".devices.";
+ switch (deviceType) {
+ case "desktop":
+ hid = "WEAVE_DEVICE_COUNT_DESKTOP";
+ prefName += "desktop";
+ break;
+ case "mobile":
+ hid = "WEAVE_DEVICE_COUNT_MOBILE";
+ prefName += "mobile";
+ break;
+ default:
+ this._log.warn(`Unexpected deviceType "${deviceType}" recording device telemetry.`);
+ continue;
+ }
+ Services.telemetry.getHistogramById(hid).add(count);
+ Svc.Prefs.set(prefName, count);
+ }
+ SyncEngine.prototype._syncFinish.call(this);
+ },
+
+ _reconcile: function _reconcile(item) {
+ // Every incoming record is reconciled, so we use this to track the
+ // contents of the collection on the server.
+ this._incomingClients[item.id] = item.modified;
+
+ if (!this._store.itemExists(item.id)) {
+ return true;
+ }
+ // Clients are synced unconditionally, so we'll always have new records.
+ // Unfortunately, this will cause the scheduler to use the immediate sync
+ // interval for the multi-device case, instead of the active interval. We
+ // work around this by updating the record during reconciliation, and
+ // returning false to indicate that the record doesn't need to be applied
+ // later.
+ this._store.update(item);
+ return false;
+ },
+
+ // Treat reset the same as wiping for locally cached clients
+ _resetClient() {
+ this._wipeClient();
+ },
+
+ _wipeClient: function _wipeClient() {
+ SyncEngine.prototype._resetClient.call(this);
+ delete this.localCommands;
+ this._store.wipe();
+ const logRemoveError = err => this._log.warn("Could not delete json file", err);
+ Async.promiseSpinningly(
+ Utils.jsonRemove("commands", this).catch(logRemoveError)
+ .then(Utils.jsonRemove("commands-syncing", this).catch(logRemoveError))
+ );
+ },
+
+ removeClientData: function removeClientData() {
+ let res = this.service.resource(this.engineURL + "/" + this.localID);
+ res.delete();
+ },
+
+ // Override the default behavior to delete bad records from the server.
+ handleHMACMismatch: function handleHMACMismatch(item, mayRetry) {
+ this._log.debug("Handling HMAC mismatch for " + item.id);
+
+ let base = SyncEngine.prototype.handleHMACMismatch.call(this, item, mayRetry);
+ if (base != SyncEngine.kRecoveryStrategy.error)
+ return base;
+
+ // It's a bad client record. Save it to be deleted at the end of the sync.
+ this._log.debug("Bad client record detected. Scheduling for deletion.");
+ this._deleteId(item.id);
+
+ // Neither try again nor error; we're going to delete it.
+ return SyncEngine.kRecoveryStrategy.ignore;
+ },
+
+ /**
+ * A hash of valid commands that the client knows about. The key is a command
+ * and the value is a hash containing information about the command such as
+ * number of arguments and description.
+ */
+ _commands: {
+ resetAll: { args: 0, desc: "Clear temporary local data for all engines" },
+ resetEngine: { args: 1, desc: "Clear temporary local data for engine" },
+ wipeAll: { args: 0, desc: "Delete all client data for all engines" },
+ wipeEngine: { args: 1, desc: "Delete all client data for engine" },
+ logout: { args: 0, desc: "Log out client" },
+ displayURI: { args: 3, desc: "Instruct a client to display a URI" },
+ },
+
+ /**
+ * Sends a command+args pair to a specific client.
+ *
+ * @param command Command string
+ * @param args Array of arguments/data for command
+ * @param clientId Client to send command to
+ */
+ _sendCommandToClient: function sendCommandToClient(command, args, clientId) {
+ this._log.trace("Sending " + command + " to " + clientId);
+
+ let client = this._store._remoteClients[clientId];
+ if (!client) {
+ throw new Error("Unknown remote client ID: '" + clientId + "'.");
+ }
+ if (client.stale) {
+ throw new Error("Stale remote client ID: '" + clientId + "'.");
+ }
+
+ let action = {
+ command: command,
+ args: args,
+ };
+
+ this._log.trace("Client " + clientId + " got a new action: " + [command, args]);
+ this._addClientCommand(clientId, action);
+ this._tracker.addChangedID(clientId);
+ },
+
+ /**
+ * Check if the local client has any remote commands and perform them.
+ *
+ * @return false to abort sync
+ */
+ processIncomingCommands: function processIncomingCommands() {
+ return this._notify("clients:process-commands", "", function() {
+ if (!this.localCommands) {
+ return true;
+ }
+
+ const clearedCommands = this._readCommands()[this.localID];
+ const commands = this.localCommands.filter(command => !hasDupeCommand(clearedCommands, command));
+
+ let URIsToDisplay = [];
+ // Process each command in order.
+ for (let rawCommand of commands) {
+ let {command, args} = rawCommand;
+ this._log.debug("Processing command: " + command + "(" + args + ")");
+
+ let engines = [args[0]];
+ switch (command) {
+ case "resetAll":
+ engines = null;
+ // Fallthrough
+ case "resetEngine":
+ this.service.resetClient(engines);
+ break;
+ case "wipeAll":
+ engines = null;
+ // Fallthrough
+ case "wipeEngine":
+ this.service.wipeClient(engines);
+ break;
+ case "logout":
+ this.service.logout();
+ return false;
+ case "displayURI":
+ let [uri, clientId, title] = args;
+ URIsToDisplay.push({ uri, clientId, title });
+ break;
+ default:
+ this._log.debug("Received an unknown command: " + command);
+ break;
+ }
+ // Add the command to the "cleared" commands list
+ this._addClientCommand(this.localID, rawCommand)
+ }
+ this._tracker.addChangedID(this.localID);
+
+ if (URIsToDisplay.length) {
+ this._handleDisplayURIs(URIsToDisplay);
+ }
+
+ return true;
+ })();
+ },
+
+ /**
+ * Validates and sends a command to a client or all clients.
+ *
+ * Calling this does not actually sync the command data to the server. If the
+ * client already has the command/args pair, it won't receive a duplicate
+ * command.
+ *
+ * @param command
+ * Command to invoke on remote clients
+ * @param args
+ * Array of arguments to give to the command
+ * @param clientId
+ * Client ID to send command to. If undefined, send to all remote
+ * clients.
+ */
+ sendCommand: function sendCommand(command, args, clientId) {
+ let commandData = this._commands[command];
+ // Don't send commands that we don't know about.
+ if (!commandData) {
+ this._log.error("Unknown command to send: " + command);
+ return;
+ }
+ // Don't send a command with the wrong number of arguments.
+ else if (!args || args.length != commandData.args) {
+ this._log.error("Expected " + commandData.args + " args for '" +
+ command + "', but got " + args);
+ return;
+ }
+
+ if (clientId) {
+ this._sendCommandToClient(command, args, clientId);
+ } else {
+ for (let [id, record] of Object.entries(this._store._remoteClients)) {
+ if (!record.stale) {
+ this._sendCommandToClient(command, args, id);
+ }
+ }
+ }
+ },
+
+ /**
+ * Send a URI to another client for display.
+ *
+ * A side effect is the score is increased dramatically to incur an
+ * immediate sync.
+ *
+ * If an unknown client ID is specified, sendCommand() will throw an
+ * Error object.
+ *
+ * @param uri
+ * URI (as a string) to send and display on the remote client
+ * @param clientId
+ * ID of client to send the command to. If not defined, will be sent
+ * to all remote clients.
+ * @param title
+ * Title of the page being sent.
+ */
+ sendURIToClientForDisplay: function sendURIToClientForDisplay(uri, clientId, title) {
+ this._log.info("Sending URI to client: " + uri + " -> " +
+ clientId + " (" + title + ")");
+ this.sendCommand("displayURI", [uri, this.localID, title], clientId);
+
+ this._tracker.score += SCORE_INCREMENT_XLARGE;
+ },
+
+ /**
+ * Handle a bunch of received 'displayURI' commands.
+ *
+ * Interested parties should observe the "weave:engine:clients:display-uris"
+ * topic. The callback will receive an array as the subject parameter
+ * containing objects with the following keys:
+ *
+ * uri URI (string) that is requested for display.
+ * clientId ID of client that sent the command.
+ * title Title of page that loaded URI (likely) corresponds to.
+ *
+ * The 'data' parameter to the callback will not be defined.
+ *
+ * @param uris
+ * An array containing URI objects to display
+ * @param uris[].uri
+ * String URI that was received
+ * @param uris[].clientId
+ * ID of client that sent URI
+ * @param uris[].title
+ * String title of page that URI corresponds to. Older clients may not
+ * send this.
+ */
+ _handleDisplayURIs: function _handleDisplayURIs(uris) {
+ Svc.Obs.notify("weave:engine:clients:display-uris", uris);
+ },
+
+ _removeRemoteClient(id) {
+ delete this._store._remoteClients[id];
+ this._tracker.removeChangedID(id);
+ },
+};
+
+function ClientStore(name, engine) {
+ Store.call(this, name, engine);
+}
+ClientStore.prototype = {
+ __proto__: Store.prototype,
+
+ _remoteClients: {},
+
+ create(record) {
+ this.update(record);
+ },
+
+ update: function update(record) {
+ if (record.id == this.engine.localID) {
+ // Only grab commands from the server; local name/type always wins
+ this.engine.localCommands = record.commands;
+ } else {
+ this._remoteClients[record.id] = record.cleartext;
+ }
+ },
+
+ createRecord: function createRecord(id, collection) {
+ let record = new ClientsRec(collection, id);
+
+ const commandsChanges = this.engine._currentlySyncingCommands ?
+ this.engine._currentlySyncingCommands[id] :
+ [];
+
+ // Package the individual components into a record for the local client
+ if (id == this.engine.localID) {
+ let cb = Async.makeSpinningCallback();
+ fxAccounts.getDeviceId().then(id => cb(null, id), cb);
+ try {
+ record.fxaDeviceId = cb.wait();
+ } catch(error) {
+ this._log.warn("failed to get fxa device id", error);
+ }
+ record.name = this.engine.localName;
+ record.type = this.engine.localType;
+ record.version = Services.appinfo.version;
+ record.protocols = SUPPORTED_PROTOCOL_VERSIONS;
+
+ // Substract the commands we recorded that we've already executed
+ if (commandsChanges && commandsChanges.length &&
+ this.engine.localCommands && this.engine.localCommands.length) {
+ record.commands = this.engine.localCommands.filter(command => !hasDupeCommand(commandsChanges, command));
+ }
+
+ // Optional fields.
+ record.os = Services.appinfo.OS; // "Darwin"
+ record.appPackage = Services.appinfo.ID;
+ record.application = this.engine.brandName // "Nightly"
+
+ // We can't compute these yet.
+ // record.device = ""; // Bug 1100723
+ // record.formfactor = ""; // Bug 1100722
+ } else {
+ record.cleartext = this._remoteClients[id];
+
+ // Add the commands we have to send
+ if (commandsChanges && commandsChanges.length) {
+ const recordCommands = record.cleartext.commands || [];
+ const newCommands = commandsChanges.filter(command => !hasDupeCommand(recordCommands, command));
+ record.cleartext.commands = recordCommands.concat(newCommands);
+ }
+
+ if (record.cleartext.stale) {
+ // It's almost certainly a logic error for us to upload a record we
+ // consider stale, so make log noise, but still remove the flag.
+ this._log.error(`Preparing to upload record ${id} that we consider stale`);
+ delete record.cleartext.stale;
+ }
+ }
+
+ return record;
+ },
+
+ itemExists(id) {
+ return id in this.getAllIDs();
+ },
+
+ getAllIDs: function getAllIDs() {
+ let ids = {};
+ ids[this.engine.localID] = true;
+ for (let id in this._remoteClients)
+ ids[id] = true;
+ return ids;
+ },
+
+ wipe: function wipe() {
+ this._remoteClients = {};
+ },
+};
+
+function ClientsTracker(name, engine) {
+ Tracker.call(this, name, engine);
+ Svc.Obs.add("weave:engine:start-tracking", this);
+ Svc.Obs.add("weave:engine:stop-tracking", this);
+}
+ClientsTracker.prototype = {
+ __proto__: Tracker.prototype,
+
+ _enabled: false,
+
+ observe: function observe(subject, topic, data) {
+ switch (topic) {
+ case "weave:engine:start-tracking":
+ if (!this._enabled) {
+ Svc.Prefs.observe("client.name", this);
+ this._enabled = true;
+ }
+ break;
+ case "weave:engine:stop-tracking":
+ if (this._enabled) {
+ Svc.Prefs.ignore("client.name", this);
+ this._enabled = false;
+ }
+ break;
+ case "nsPref:changed":
+ this._log.debug("client.name preference changed");
+ this.addChangedID(Svc.Prefs.get("client.GUID"));
+ this.score += SCORE_INCREMENT_XLARGE;
+ break;
+ }
+ }
+};
diff --git a/services/sync/modules/engines/extension-storage.js b/services/sync/modules/engines/extension-storage.js
new file mode 100644
index 000000000..f8f15b128
--- /dev/null
+++ b/services/sync/modules/engines/extension-storage.js
@@ -0,0 +1,277 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+this.EXPORTED_SYMBOLS = ['ExtensionStorageEngine', 'EncryptionRemoteTransformer',
+ 'KeyRingEncryptionRemoteTransformer'];
+
+const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
+
+Cu.import("resource://services-crypto/utils.js");
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://services-sync/engines.js");
+Cu.import("resource://services-sync/keys.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://services-common/async.js");
+XPCOMUtils.defineLazyModuleGetter(this, "ExtensionStorageSync",
+ "resource://gre/modules/ExtensionStorageSync.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "fxAccounts",
+ "resource://gre/modules/FxAccounts.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "Task",
+ "resource://gre/modules/Task.jsm");
+
+/**
+ * The Engine that manages syncing for the web extension "storage"
+ * API, and in particular ext.storage.sync.
+ *
+ * ext.storage.sync is implemented using Kinto, so it has mechanisms
+ * for syncing that we do not need to integrate in the Firefox Sync
+ * framework, so this is something of a stub.
+ */
+this.ExtensionStorageEngine = function ExtensionStorageEngine(service) {
+ SyncEngine.call(this, "Extension-Storage", service);
+};
+ExtensionStorageEngine.prototype = {
+ __proto__: SyncEngine.prototype,
+ _trackerObj: ExtensionStorageTracker,
+ // we don't need these since we implement our own sync logic
+ _storeObj: undefined,
+ _recordObj: undefined,
+
+ syncPriority: 10,
+ allowSkippedRecord: false,
+
+ _sync: function () {
+ return Async.promiseSpinningly(ExtensionStorageSync.syncAll());
+ },
+
+ get enabled() {
+ // By default, we sync extension storage if we sync addons. This
+ // lets us simplify the UX since users probably don't consider
+ // "extension preferences" a separate category of syncing.
+ // However, we also respect engine.extension-storage.force, which
+ // can be set to true or false, if a power user wants to customize
+ // the behavior despite the lack of UI.
+ const forced = Svc.Prefs.get("engine." + this.prefName + ".force", undefined);
+ if (forced !== undefined) {
+ return forced;
+ }
+ return Svc.Prefs.get("engine.addons", false);
+ },
+};
+
+function ExtensionStorageTracker(name, engine) {
+ Tracker.call(this, name, engine);
+}
+ExtensionStorageTracker.prototype = {
+ __proto__: Tracker.prototype,
+
+ startTracking: function () {
+ Svc.Obs.add("ext.storage.sync-changed", this);
+ },
+
+ stopTracking: function () {
+ Svc.Obs.remove("ext.storage.sync-changed", this);
+ },
+
+ observe: function (subject, topic, data) {
+ Tracker.prototype.observe.call(this, subject, topic, data);
+
+ if (this.ignoreAll) {
+ return;
+ }
+
+ if (topic !== "ext.storage.sync-changed") {
+ return;
+ }
+
+ // Single adds, removes and changes are not so important on their
+ // own, so let's just increment score a bit.
+ this.score += SCORE_INCREMENT_MEDIUM;
+ },
+
+ // Override a bunch of methods which don't do anything for us.
+ // This is a performance hack.
+ saveChangedIDs: function() {
+ },
+ loadChangedIDs: function() {
+ },
+ ignoreID: function() {
+ },
+ unignoreID: function() {
+ },
+ addChangedID: function() {
+ },
+ removeChangedID: function() {
+ },
+ clearChangedIDs: function() {
+ },
+};
+
+/**
+ * Utility function to enforce an order of fields when computing an HMAC.
+ */
+function ciphertextHMAC(keyBundle, id, IV, ciphertext) {
+ const hasher = keyBundle.sha256HMACHasher;
+ return Utils.bytesAsHex(Utils.digestUTF8(id + IV + ciphertext, hasher));
+}
+
+/**
+ * A "remote transformer" that the Kinto library will use to
+ * encrypt/decrypt records when syncing.
+ *
+ * This is an "abstract base class". Subclass this and override
+ * getKeys() to use it.
+ */
+class EncryptionRemoteTransformer {
+ encode(record) {
+ const self = this;
+ return Task.spawn(function* () {
+ const keyBundle = yield self.getKeys();
+ if (record.ciphertext) {
+ throw new Error("Attempt to reencrypt??");
+ }
+ let id = record.id;
+ if (!record.id) {
+ throw new Error("Record ID is missing or invalid");
+ }
+
+ let IV = Svc.Crypto.generateRandomIV();
+ let ciphertext = Svc.Crypto.encrypt(JSON.stringify(record),
+ keyBundle.encryptionKeyB64, IV);
+ let hmac = ciphertextHMAC(keyBundle, id, IV, ciphertext);
+ const encryptedResult = {ciphertext, IV, hmac, id};
+ if (record.hasOwnProperty("last_modified")) {
+ encryptedResult.last_modified = record.last_modified;
+ }
+ return encryptedResult;
+ });
+ }
+
+ decode(record) {
+ const self = this;
+ return Task.spawn(function* () {
+ if (!record.ciphertext) {
+ // This can happen for tombstones if a record is deleted.
+ if (record.deleted) {
+ return record;
+ }
+ throw new Error("No ciphertext: nothing to decrypt?");
+ }
+ const keyBundle = yield self.getKeys();
+ // Authenticate the encrypted blob with the expected HMAC
+ let computedHMAC = ciphertextHMAC(keyBundle, record.id, record.IV, record.ciphertext);
+
+ if (computedHMAC != record.hmac) {
+ Utils.throwHMACMismatch(record.hmac, computedHMAC);
+ }
+
+ // Handle invalid data here. Elsewhere we assume that cleartext is an object.
+ let cleartext = Svc.Crypto.decrypt(record.ciphertext,
+ keyBundle.encryptionKeyB64, record.IV);
+ let jsonResult = JSON.parse(cleartext);
+ if (!jsonResult || typeof jsonResult !== "object") {
+ throw new Error("Decryption failed: result is <" + jsonResult + ">, not an object.");
+ }
+
+ // Verify that the encrypted id matches the requested record's id.
+ // This should always be true, because we compute the HMAC over
+ // the original record's ID, and that was verified already (above).
+ if (jsonResult.id != record.id) {
+ throw new Error("Record id mismatch: " + jsonResult.id + " != " + record.id);
+ }
+
+ if (record.hasOwnProperty("last_modified")) {
+ jsonResult.last_modified = record.last_modified;
+ }
+
+ return jsonResult;
+ });
+ }
+
+ /**
+ * Retrieve keys to use during encryption.
+ *
+ * Returns a Promise<KeyBundle>.
+ */
+ getKeys() {
+ throw new Error("override getKeys in a subclass");
+ }
+}
+// You can inject this
+EncryptionRemoteTransformer.prototype._fxaService = fxAccounts;
+
+/**
+ * An EncryptionRemoteTransformer that provides a keybundle derived
+ * from the user's kB, suitable for encrypting a keyring.
+ */
+class KeyRingEncryptionRemoteTransformer extends EncryptionRemoteTransformer {
+ getKeys() {
+ const self = this;
+ return Task.spawn(function* () {
+ const user = yield self._fxaService.getSignedInUser();
+ // FIXME: we should permit this if the user is self-hosting
+ // their storage
+ if (!user) {
+ throw new Error("user isn't signed in to FxA; can't sync");
+ }
+
+ if (!user.kB) {
+ throw new Error("user doesn't have kB");
+ }
+
+ let kB = Utils.hexToBytes(user.kB);
+
+ let keyMaterial = CryptoUtils.hkdf(kB, undefined,
+ "identity.mozilla.com/picl/v1/chrome.storage.sync", 2*32);
+ let bundle = new BulkKeyBundle();
+ // [encryptionKey, hmacKey]
+ bundle.keyPair = [keyMaterial.slice(0, 32), keyMaterial.slice(32, 64)];
+ return bundle;
+ });
+ }
+ // Pass through the kbHash field from the unencrypted record. If
+ // encryption fails, we can use this to try to detect whether we are
+ // being compromised or if the record here was encoded with a
+ // different kB.
+ encode(record) {
+ const encodePromise = super.encode(record);
+ return Task.spawn(function* () {
+ const encoded = yield encodePromise;
+ encoded.kbHash = record.kbHash;
+ return encoded;
+ });
+ }
+
+ decode(record) {
+ const decodePromise = super.decode(record);
+ return Task.spawn(function* () {
+ try {
+ return yield decodePromise;
+ } catch (e) {
+ if (Utils.isHMACMismatch(e)) {
+ const currentKBHash = yield ExtensionStorageSync.getKBHash();
+ if (record.kbHash != currentKBHash) {
+ // Some other client encoded this with a kB that we don't
+ // have access to.
+ KeyRingEncryptionRemoteTransformer.throwOutdatedKB(currentKBHash, record.kbHash);
+ }
+ }
+ throw e;
+ }
+ });
+ }
+
+ // Generator and discriminator for KB-is-outdated exceptions.
+ static throwOutdatedKB(shouldBe, is) {
+ throw new Error(`kB hash on record is outdated: should be ${shouldBe}, is ${is}`);
+ }
+
+ static isOutdatedKB(exc) {
+ const kbMessage = "kB hash on record is outdated: ";
+ return exc && exc.message && exc.message.indexOf &&
+ (exc.message.indexOf(kbMessage) == 0);
+ }
+}
diff --git a/services/sync/modules/engines/forms.js b/services/sync/modules/engines/forms.js
new file mode 100644
index 000000000..43f79d4f7
--- /dev/null
+++ b/services/sync/modules/engines/forms.js
@@ -0,0 +1,305 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ['FormEngine', 'FormRec', 'FormValidator'];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cu = Components.utils;
+
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+Cu.import("resource://services-sync/engines.js");
+Cu.import("resource://services-sync/record.js");
+Cu.import("resource://services-common/async.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://services-sync/collection_validator.js");
+Cu.import("resource://gre/modules/Log.jsm");
+
+const FORMS_TTL = 3 * 365 * 24 * 60 * 60; // Three years in seconds.
+
+this.FormRec = function FormRec(collection, id) {
+ CryptoWrapper.call(this, collection, id);
+}
+FormRec.prototype = {
+ __proto__: CryptoWrapper.prototype,
+ _logName: "Sync.Record.Form",
+ ttl: FORMS_TTL
+};
+
+Utils.deferGetSet(FormRec, "cleartext", ["name", "value"]);
+
+
+var FormWrapper = {
+ _log: Log.repository.getLogger("Sync.Engine.Forms"),
+
+ _getEntryCols: ["fieldname", "value"],
+ _guidCols: ["guid"],
+
+ _promiseSearch: function(terms, searchData) {
+ return new Promise(resolve => {
+ let results = [];
+ let callbacks = {
+ handleResult(result) {
+ results.push(result);
+ },
+ handleCompletion(reason) {
+ resolve(results);
+ }
+ };
+ Svc.FormHistory.search(terms, searchData, callbacks);
+ })
+ },
+
+ // Do a "sync" search by spinning the event loop until it completes.
+ _searchSpinningly: function(terms, searchData) {
+ return Async.promiseSpinningly(this._promiseSearch(terms, searchData));
+ },
+
+ _updateSpinningly: function(changes) {
+ if (!Svc.FormHistory.enabled) {
+ return; // update isn't going to do anything.
+ }
+ let cb = Async.makeSpinningCallback();
+ let callbacks = {
+ handleCompletion: function(reason) {
+ cb();
+ }
+ };
+ Svc.FormHistory.update(changes, callbacks);
+ return cb.wait();
+ },
+
+ getEntry: function (guid) {
+ let results = this._searchSpinningly(this._getEntryCols, {guid: guid});
+ if (!results.length) {
+ return null;
+ }
+ return {name: results[0].fieldname, value: results[0].value};
+ },
+
+ getGUID: function (name, value) {
+ // Query for the provided entry.
+ let query = { fieldname: name, value: value };
+ let results = this._searchSpinningly(this._guidCols, query);
+ return results.length ? results[0].guid : null;
+ },
+
+ hasGUID: function (guid) {
+ // We could probably use a count function here, but searchSpinningly exists...
+ return this._searchSpinningly(this._guidCols, {guid: guid}).length != 0;
+ },
+
+ replaceGUID: function (oldGUID, newGUID) {
+ let changes = {
+ op: "update",
+ guid: oldGUID,
+ newGuid: newGUID,
+ }
+ this._updateSpinningly(changes);
+ }
+
+};
+
+this.FormEngine = function FormEngine(service) {
+ SyncEngine.call(this, "Forms", service);
+}
+FormEngine.prototype = {
+ __proto__: SyncEngine.prototype,
+ _storeObj: FormStore,
+ _trackerObj: FormTracker,
+ _recordObj: FormRec,
+ applyIncomingBatchSize: FORMS_STORE_BATCH_SIZE,
+
+ syncPriority: 6,
+
+ get prefName() {
+ return "history";
+ },
+
+ _findDupe: function _findDupe(item) {
+ return FormWrapper.getGUID(item.name, item.value);
+ }
+};
+
+function FormStore(name, engine) {
+ Store.call(this, name, engine);
+}
+FormStore.prototype = {
+ __proto__: Store.prototype,
+
+ _processChange: function (change) {
+ // If this._changes is defined, then we are applying a batch, so we
+ // can defer it.
+ if (this._changes) {
+ this._changes.push(change);
+ return;
+ }
+
+ // Otherwise we must handle the change synchronously, right now.
+ FormWrapper._updateSpinningly(change);
+ },
+
+ applyIncomingBatch: function (records) {
+ // We collect all the changes to be made then apply them all at once.
+ this._changes = [];
+ let failures = Store.prototype.applyIncomingBatch.call(this, records);
+ if (this._changes.length) {
+ FormWrapper._updateSpinningly(this._changes);
+ }
+ delete this._changes;
+ return failures;
+ },
+
+ getAllIDs: function () {
+ let results = FormWrapper._searchSpinningly(["guid"], [])
+ let guids = {};
+ for (let result of results) {
+ guids[result.guid] = true;
+ }
+ return guids;
+ },
+
+ changeItemID: function (oldID, newID) {
+ FormWrapper.replaceGUID(oldID, newID);
+ },
+
+ itemExists: function (id) {
+ return FormWrapper.hasGUID(id);
+ },
+
+ createRecord: function (id, collection) {
+ let record = new FormRec(collection, id);
+ let entry = FormWrapper.getEntry(id);
+ if (entry != null) {
+ record.name = entry.name;
+ record.value = entry.value;
+ } else {
+ record.deleted = true;
+ }
+ return record;
+ },
+
+ create: function (record) {
+ this._log.trace("Adding form record for " + record.name);
+ let change = {
+ op: "add",
+ fieldname: record.name,
+ value: record.value
+ };
+ this._processChange(change);
+ },
+
+ remove: function (record) {
+ this._log.trace("Removing form record: " + record.id);
+ let change = {
+ op: "remove",
+ guid: record.id
+ };
+ this._processChange(change);
+ },
+
+ update: function (record) {
+ this._log.trace("Ignoring form record update request!");
+ },
+
+ wipe: function () {
+ let change = {
+ op: "remove"
+ };
+ FormWrapper._updateSpinningly(change);
+ }
+};
+
+function FormTracker(name, engine) {
+ Tracker.call(this, name, engine);
+}
+FormTracker.prototype = {
+ __proto__: Tracker.prototype,
+
+ QueryInterface: XPCOMUtils.generateQI([
+ Ci.nsIObserver,
+ Ci.nsISupportsWeakReference]),
+
+ startTracking: function() {
+ Svc.Obs.add("satchel-storage-changed", this);
+ },
+
+ stopTracking: function() {
+ Svc.Obs.remove("satchel-storage-changed", this);
+ },
+
+ observe: function (subject, topic, data) {
+ Tracker.prototype.observe.call(this, subject, topic, data);
+ if (this.ignoreAll) {
+ return;
+ }
+ switch (topic) {
+ case "satchel-storage-changed":
+ if (data == "formhistory-add" || data == "formhistory-remove") {
+ let guid = subject.QueryInterface(Ci.nsISupportsString).toString();
+ this.trackEntry(guid);
+ }
+ break;
+ }
+ },
+
+ trackEntry: function (guid) {
+ this.addChangedID(guid);
+ this.score += SCORE_INCREMENT_MEDIUM;
+ },
+};
+
+
+class FormsProblemData extends CollectionProblemData {
+ getSummary() {
+ // We don't support syncing deleted form data, so "clientMissing" isn't a problem
+ return super.getSummary().filter(entry =>
+ entry.name !== "clientMissing");
+ }
+}
+
+class FormValidator extends CollectionValidator {
+ constructor() {
+ super("forms", "id", ["name", "value"]);
+ }
+
+ emptyProblemData() {
+ return new FormsProblemData();
+ }
+
+ getClientItems() {
+ return FormWrapper._promiseSearch(["guid", "fieldname", "value"], {});
+ }
+
+ normalizeClientItem(item) {
+ return {
+ id: item.guid,
+ guid: item.guid,
+ name: item.fieldname,
+ fieldname: item.fieldname,
+ value: item.value,
+ original: item,
+ };
+ }
+
+ normalizeServerItem(item) {
+ let res = Object.assign({
+ guid: item.id,
+ fieldname: item.name,
+ original: item,
+ }, item);
+ // Missing `name` or `value` causes the getGUID call to throw
+ if (item.name !== undefined && item.value !== undefined) {
+ let guid = FormWrapper.getGUID(item.name, item.value);
+ if (guid) {
+ res.guid = guid;
+ res.id = guid;
+ res.duped = true;
+ }
+ }
+
+ return res;
+ }
+} \ No newline at end of file
diff --git a/services/sync/modules/engines/history.js b/services/sync/modules/engines/history.js
new file mode 100644
index 000000000..307d484c1
--- /dev/null
+++ b/services/sync/modules/engines/history.js
@@ -0,0 +1,442 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ['HistoryEngine', 'HistoryRec'];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cu = Components.utils;
+var Cr = Components.results;
+
+const HISTORY_TTL = 5184000; // 60 days
+
+Cu.import("resource://gre/modules/PlacesUtils.jsm", this);
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+Cu.import("resource://services-common/async.js");
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://services-sync/engines.js");
+Cu.import("resource://services-sync/record.js");
+Cu.import("resource://services-sync/util.js");
+
+this.HistoryRec = function HistoryRec(collection, id) {
+ CryptoWrapper.call(this, collection, id);
+}
+HistoryRec.prototype = {
+ __proto__: CryptoWrapper.prototype,
+ _logName: "Sync.Record.History",
+ ttl: HISTORY_TTL
+};
+
+Utils.deferGetSet(HistoryRec, "cleartext", ["histUri", "title", "visits"]);
+
+
+this.HistoryEngine = function HistoryEngine(service) {
+ SyncEngine.call(this, "History", service);
+}
+HistoryEngine.prototype = {
+ __proto__: SyncEngine.prototype,
+ _recordObj: HistoryRec,
+ _storeObj: HistoryStore,
+ _trackerObj: HistoryTracker,
+ downloadLimit: MAX_HISTORY_DOWNLOAD,
+ applyIncomingBatchSize: HISTORY_STORE_BATCH_SIZE,
+
+ syncPriority: 7,
+
+ _processIncoming: function (newitems) {
+ // We want to notify history observers that a batch operation is underway
+ // so they don't do lots of work for each incoming record.
+ let observers = PlacesUtils.history.getObservers();
+ function notifyHistoryObservers(notification) {
+ for (let observer of observers) {
+ try {
+ observer[notification]();
+ } catch (ex) { }
+ }
+ }
+ notifyHistoryObservers("onBeginUpdateBatch");
+ try {
+ return SyncEngine.prototype._processIncoming.call(this, newitems);
+ } finally {
+ notifyHistoryObservers("onEndUpdateBatch");
+ }
+ },
+};
+
+function HistoryStore(name, engine) {
+ Store.call(this, name, engine);
+
+ // Explicitly nullify our references to our cached services so we don't leak
+ Svc.Obs.add("places-shutdown", function() {
+ for (let query in this._stmts) {
+ let stmt = this._stmts;
+ stmt.finalize();
+ }
+ this._stmts = {};
+ }, this);
+}
+HistoryStore.prototype = {
+ __proto__: Store.prototype,
+
+ __asyncHistory: null,
+ get _asyncHistory() {
+ if (!this.__asyncHistory) {
+ this.__asyncHistory = Cc["@mozilla.org/browser/history;1"]
+ .getService(Ci.mozIAsyncHistory);
+ }
+ return this.__asyncHistory;
+ },
+
+ _stmts: {},
+ _getStmt: function(query) {
+ if (query in this._stmts) {
+ return this._stmts[query];
+ }
+
+ this._log.trace("Creating SQL statement: " + query);
+ let db = PlacesUtils.history.QueryInterface(Ci.nsPIPlacesDatabase)
+ .DBConnection;
+ return this._stmts[query] = db.createAsyncStatement(query);
+ },
+
+ get _setGUIDStm() {
+ return this._getStmt(
+ "UPDATE moz_places " +
+ "SET guid = :guid " +
+ "WHERE url_hash = hash(:page_url) AND url = :page_url");
+ },
+
+ // Some helper functions to handle GUIDs
+ setGUID: function setGUID(uri, guid) {
+ uri = uri.spec ? uri.spec : uri;
+
+ if (!guid) {
+ guid = Utils.makeGUID();
+ }
+
+ let stmt = this._setGUIDStm;
+ stmt.params.guid = guid;
+ stmt.params.page_url = uri;
+ Async.querySpinningly(stmt);
+ return guid;
+ },
+
+ get _guidStm() {
+ return this._getStmt(
+ "SELECT guid " +
+ "FROM moz_places " +
+ "WHERE url_hash = hash(:page_url) AND url = :page_url");
+ },
+ _guidCols: ["guid"],
+
+ GUIDForUri: function GUIDForUri(uri, create) {
+ let stm = this._guidStm;
+ stm.params.page_url = uri.spec ? uri.spec : uri;
+
+ // Use the existing GUID if it exists
+ let result = Async.querySpinningly(stm, this._guidCols)[0];
+ if (result && result.guid)
+ return result.guid;
+
+ // Give the uri a GUID if it doesn't have one
+ if (create)
+ return this.setGUID(uri);
+ },
+
+ get _visitStm() {
+ return this._getStmt(`/* do not warn (bug 599936) */
+ SELECT visit_type type, visit_date date
+ FROM moz_historyvisits
+ JOIN moz_places h ON h.id = place_id
+ WHERE url_hash = hash(:url) AND url = :url
+ ORDER BY date DESC LIMIT 20`);
+ },
+ _visitCols: ["date", "type"],
+
+ get _urlStm() {
+ return this._getStmt(
+ "SELECT url, title, frecency " +
+ "FROM moz_places " +
+ "WHERE guid = :guid");
+ },
+ _urlCols: ["url", "title", "frecency"],
+
+ get _allUrlStm() {
+ return this._getStmt(
+ "SELECT url " +
+ "FROM moz_places " +
+ "WHERE last_visit_date > :cutoff_date " +
+ "ORDER BY frecency DESC " +
+ "LIMIT :max_results");
+ },
+ _allUrlCols: ["url"],
+
+ // See bug 320831 for why we use SQL here
+ _getVisits: function HistStore__getVisits(uri) {
+ this._visitStm.params.url = uri;
+ return Async.querySpinningly(this._visitStm, this._visitCols);
+ },
+
+ // See bug 468732 for why we use SQL here
+ _findURLByGUID: function HistStore__findURLByGUID(guid) {
+ this._urlStm.params.guid = guid;
+ return Async.querySpinningly(this._urlStm, this._urlCols)[0];
+ },
+
+ changeItemID: function HStore_changeItemID(oldID, newID) {
+ this.setGUID(this._findURLByGUID(oldID).url, newID);
+ },
+
+
+ getAllIDs: function HistStore_getAllIDs() {
+ // Only get places visited within the last 30 days (30*24*60*60*1000ms)
+ this._allUrlStm.params.cutoff_date = (Date.now() - 2592000000) * 1000;
+ this._allUrlStm.params.max_results = MAX_HISTORY_UPLOAD;
+
+ let urls = Async.querySpinningly(this._allUrlStm, this._allUrlCols);
+ let self = this;
+ return urls.reduce(function(ids, item) {
+ ids[self.GUIDForUri(item.url, true)] = item.url;
+ return ids;
+ }, {});
+ },
+
+ applyIncomingBatch: function applyIncomingBatch(records) {
+ let failed = [];
+
+ // Convert incoming records to mozIPlaceInfo objects. Some records can be
+ // ignored or handled directly, so we're rewriting the array in-place.
+ let i, k;
+ for (i = 0, k = 0; i < records.length; i++) {
+ let record = records[k] = records[i];
+ let shouldApply;
+
+ // This is still synchronous I/O for now.
+ try {
+ if (record.deleted) {
+ // Consider using nsIBrowserHistory::removePages() here.
+ this.remove(record);
+ // No further processing needed. Remove it from the list.
+ shouldApply = false;
+ } else {
+ shouldApply = this._recordToPlaceInfo(record);
+ }
+ } catch (ex) {
+ if (Async.isShutdownException(ex)) {
+ throw ex;
+ }
+ failed.push(record.id);
+ shouldApply = false;
+ }
+
+ if (shouldApply) {
+ k += 1;
+ }
+ }
+ records.length = k; // truncate array
+
+ // Nothing to do.
+ if (!records.length) {
+ return failed;
+ }
+
+ let updatePlacesCallback = {
+ handleResult: function handleResult() {},
+ handleError: function handleError(resultCode, placeInfo) {
+ failed.push(placeInfo.guid);
+ },
+ handleCompletion: Async.makeSyncCallback()
+ };
+ this._asyncHistory.updatePlaces(records, updatePlacesCallback);
+ Async.waitForSyncCallback(updatePlacesCallback.handleCompletion);
+ return failed;
+ },
+
+ /**
+ * Converts a Sync history record to a mozIPlaceInfo.
+ *
+ * Throws if an invalid record is encountered (invalid URI, etc.),
+ * returns true if the record is to be applied, false otherwise
+ * (no visits to add, etc.),
+ */
+ _recordToPlaceInfo: function _recordToPlaceInfo(record) {
+ // Sort out invalid URIs and ones Places just simply doesn't want.
+ record.uri = Utils.makeURI(record.histUri);
+ if (!record.uri) {
+ this._log.warn("Attempted to process invalid URI, skipping.");
+ throw "Invalid URI in record";
+ }
+
+ if (!Utils.checkGUID(record.id)) {
+ this._log.warn("Encountered record with invalid GUID: " + record.id);
+ return false;
+ }
+ record.guid = record.id;
+
+ if (!PlacesUtils.history.canAddURI(record.uri)) {
+ this._log.trace("Ignoring record " + record.id + " with URI "
+ + record.uri.spec + ": can't add this URI.");
+ return false;
+ }
+
+ // We dupe visits by date and type. So an incoming visit that has
+ // the same timestamp and type as a local one won't get applied.
+ // To avoid creating new objects, we rewrite the query result so we
+ // can simply check for containment below.
+ let curVisits = this._getVisits(record.histUri);
+ let i, k;
+ for (i = 0; i < curVisits.length; i++) {
+ curVisits[i] = curVisits[i].date + "," + curVisits[i].type;
+ }
+
+ // Walk through the visits, make sure we have sound data, and eliminate
+ // dupes. The latter is done by rewriting the array in-place.
+ for (i = 0, k = 0; i < record.visits.length; i++) {
+ let visit = record.visits[k] = record.visits[i];
+
+ if (!visit.date || typeof visit.date != "number") {
+ this._log.warn("Encountered record with invalid visit date: "
+ + visit.date);
+ continue;
+ }
+
+ if (!visit.type ||
+ !Object.values(PlacesUtils.history.TRANSITIONS).includes(visit.type)) {
+ this._log.warn("Encountered record with invalid visit type: " +
+ visit.type + "; ignoring.");
+ continue;
+ }
+
+ // Dates need to be integers.
+ visit.date = Math.round(visit.date);
+
+ if (curVisits.indexOf(visit.date + "," + visit.type) != -1) {
+ // Visit is a dupe, don't increment 'k' so the element will be
+ // overwritten.
+ continue;
+ }
+
+ visit.visitDate = visit.date;
+ visit.transitionType = visit.type;
+ k += 1;
+ }
+ record.visits.length = k; // truncate array
+
+ // No update if there aren't any visits to apply.
+ // mozIAsyncHistory::updatePlaces() wants at least one visit.
+ // In any case, the only thing we could change would be the title
+ // and that shouldn't change without a visit.
+ if (!record.visits.length) {
+ this._log.trace("Ignoring record " + record.id + " with URI "
+ + record.uri.spec + ": no visits to add.");
+ return false;
+ }
+
+ return true;
+ },
+
+ remove: function HistStore_remove(record) {
+ let page = this._findURLByGUID(record.id);
+ if (page == null) {
+ this._log.debug("Page already removed: " + record.id);
+ return;
+ }
+
+ let uri = Utils.makeURI(page.url);
+ PlacesUtils.history.removePage(uri);
+ this._log.trace("Removed page: " + [record.id, page.url, page.title]);
+ },
+
+ itemExists: function HistStore_itemExists(id) {
+ return !!this._findURLByGUID(id);
+ },
+
+ createRecord: function createRecord(id, collection) {
+ let foo = this._findURLByGUID(id);
+ let record = new HistoryRec(collection, id);
+ if (foo) {
+ record.histUri = foo.url;
+ record.title = foo.title;
+ record.sortindex = foo.frecency;
+ record.visits = this._getVisits(record.histUri);
+ } else {
+ record.deleted = true;
+ }
+
+ return record;
+ },
+
+ wipe: function HistStore_wipe() {
+ let cb = Async.makeSyncCallback();
+ PlacesUtils.history.clear().then(result => {cb(null, result)}, err => {cb(err)});
+ return Async.waitForSyncCallback(cb);
+ }
+};
+
+function HistoryTracker(name, engine) {
+ Tracker.call(this, name, engine);
+}
+HistoryTracker.prototype = {
+ __proto__: Tracker.prototype,
+
+ startTracking: function() {
+ this._log.info("Adding Places observer.");
+ PlacesUtils.history.addObserver(this, true);
+ },
+
+ stopTracking: function() {
+ this._log.info("Removing Places observer.");
+ PlacesUtils.history.removeObserver(this);
+ },
+
+ QueryInterface: XPCOMUtils.generateQI([
+ Ci.nsINavHistoryObserver,
+ Ci.nsISupportsWeakReference
+ ]),
+
+ onDeleteAffectsGUID: function (uri, guid, reason, source, increment) {
+ if (this.ignoreAll || reason == Ci.nsINavHistoryObserver.REASON_EXPIRED) {
+ return;
+ }
+ this._log.trace(source + ": " + uri.spec + ", reason " + reason);
+ if (this.addChangedID(guid)) {
+ this.score += increment;
+ }
+ },
+
+ onDeleteVisits: function (uri, visitTime, guid, reason) {
+ this.onDeleteAffectsGUID(uri, guid, reason, "onDeleteVisits", SCORE_INCREMENT_SMALL);
+ },
+
+ onDeleteURI: function (uri, guid, reason) {
+ this.onDeleteAffectsGUID(uri, guid, reason, "onDeleteURI", SCORE_INCREMENT_XLARGE);
+ },
+
+ onVisit: function (uri, vid, time, session, referrer, trans, guid) {
+ if (this.ignoreAll) {
+ this._log.trace("ignoreAll: ignoring visit for " + guid);
+ return;
+ }
+
+ this._log.trace("onVisit: " + uri.spec);
+ if (this.addChangedID(guid)) {
+ this.score += SCORE_INCREMENT_SMALL;
+ }
+ },
+
+ onClearHistory: function () {
+ this._log.trace("onClearHistory");
+ // Note that we're going to trigger a sync, but none of the cleared
+ // pages are tracked, so the deletions will not be propagated.
+ // See Bug 578694.
+ this.score += SCORE_INCREMENT_XLARGE;
+ },
+
+ onBeginUpdateBatch: function () {},
+ onEndUpdateBatch: function () {},
+ onPageChanged: function () {},
+ onTitleChanged: function () {},
+ onBeforeDeleteURI: function () {},
+};
diff --git a/services/sync/modules/engines/passwords.js b/services/sync/modules/engines/passwords.js
new file mode 100644
index 000000000..51db49a0a
--- /dev/null
+++ b/services/sync/modules/engines/passwords.js
@@ -0,0 +1,371 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ['PasswordEngine', 'LoginRec', 'PasswordValidator'];
+
+var {classes: Cc, interfaces: Ci, utils: Cu} = Components;
+
+Cu.import("resource://services-sync/record.js");
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://services-sync/collection_validator.js");
+Cu.import("resource://services-sync/engines.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://services-common/async.js");
+
+this.LoginRec = function LoginRec(collection, id) {
+ CryptoWrapper.call(this, collection, id);
+}
+LoginRec.prototype = {
+ __proto__: CryptoWrapper.prototype,
+ _logName: "Sync.Record.Login",
+};
+
+Utils.deferGetSet(LoginRec, "cleartext", [
+ "hostname", "formSubmitURL",
+ "httpRealm", "username", "password", "usernameField", "passwordField",
+ "timeCreated", "timePasswordChanged",
+ ]);
+
+
+this.PasswordEngine = function PasswordEngine(service) {
+ SyncEngine.call(this, "Passwords", service);
+}
+PasswordEngine.prototype = {
+ __proto__: SyncEngine.prototype,
+ _storeObj: PasswordStore,
+ _trackerObj: PasswordTracker,
+ _recordObj: LoginRec,
+
+ applyIncomingBatchSize: PASSWORDS_STORE_BATCH_SIZE,
+
+ syncPriority: 2,
+
+ _syncFinish: function () {
+ SyncEngine.prototype._syncFinish.call(this);
+
+ // Delete the Weave credentials from the server once.
+ if (!Svc.Prefs.get("deletePwdFxA", false)) {
+ try {
+ let ids = [];
+ for (let host of Utils.getSyncCredentialsHosts()) {
+ for (let info of Services.logins.findLogins({}, host, "", "")) {
+ ids.push(info.QueryInterface(Components.interfaces.nsILoginMetaInfo).guid);
+ }
+ }
+ if (ids.length) {
+ let coll = new Collection(this.engineURL, null, this.service);
+ coll.ids = ids;
+ let ret = coll.delete();
+ this._log.debug("Delete result: " + ret);
+ if (!ret.success && ret.status != 400) {
+ // A non-400 failure means try again next time.
+ return;
+ }
+ } else {
+ this._log.debug("Didn't find any passwords to delete");
+ }
+ // If there were no ids to delete, or we succeeded, or got a 400,
+ // record success.
+ Svc.Prefs.set("deletePwdFxA", true);
+ Svc.Prefs.reset("deletePwd"); // The old prefname we previously used.
+ } catch (ex) {
+ if (Async.isShutdownException(ex)) {
+ throw ex;
+ }
+ this._log.debug("Password deletes failed", ex);
+ }
+ }
+ },
+
+ _findDupe: function (item) {
+ let login = this._store._nsLoginInfoFromRecord(item);
+ if (!login) {
+ return;
+ }
+
+ let logins = Services.logins.findLogins({}, login.hostname, login.formSubmitURL, login.httpRealm);
+
+ this._store._sleep(0); // Yield back to main thread after synchronous operation.
+
+ // Look for existing logins that match the hostname, but ignore the password.
+ for (let local of logins) {
+ if (login.matches(local, true) && local instanceof Ci.nsILoginMetaInfo) {
+ return local.guid;
+ }
+ }
+ },
+};
+
+function PasswordStore(name, engine) {
+ Store.call(this, name, engine);
+ this._nsLoginInfo = new Components.Constructor("@mozilla.org/login-manager/loginInfo;1", Ci.nsILoginInfo, "init");
+}
+PasswordStore.prototype = {
+ __proto__: Store.prototype,
+
+ _newPropertyBag: function () {
+ return Cc["@mozilla.org/hash-property-bag;1"].createInstance(Ci.nsIWritablePropertyBag2);
+ },
+
+ /**
+ * Return an instance of nsILoginInfo (and, implicitly, nsILoginMetaInfo).
+ */
+ _nsLoginInfoFromRecord: function (record) {
+ function nullUndefined(x) {
+ return (x == undefined) ? null : x;
+ }
+
+ if (record.formSubmitURL && record.httpRealm) {
+ this._log.warn("Record " + record.id + " has both formSubmitURL and httpRealm. Skipping.");
+ return null;
+ }
+
+ // Passing in "undefined" results in an empty string, which later
+ // counts as a value. Explicitly `|| null` these fields according to JS
+ // truthiness. Records with empty strings or null will be unmolested.
+ let info = new this._nsLoginInfo(record.hostname,
+ nullUndefined(record.formSubmitURL),
+ nullUndefined(record.httpRealm),
+ record.username,
+ record.password,
+ record.usernameField,
+ record.passwordField);
+
+ info.QueryInterface(Ci.nsILoginMetaInfo);
+ info.guid = record.id;
+ if (record.timeCreated) {
+ info.timeCreated = record.timeCreated;
+ }
+ if (record.timePasswordChanged) {
+ info.timePasswordChanged = record.timePasswordChanged;
+ }
+
+ return info;
+ },
+
+ _getLoginFromGUID: function (id) {
+ let prop = this._newPropertyBag();
+ prop.setPropertyAsAUTF8String("guid", id);
+
+ let logins = Services.logins.searchLogins({}, prop);
+ this._sleep(0); // Yield back to main thread after synchronous operation.
+
+ if (logins.length > 0) {
+ this._log.trace(logins.length + " items matching " + id + " found.");
+ return logins[0];
+ }
+
+ this._log.trace("No items matching " + id + " found. Ignoring");
+ return null;
+ },
+
+ getAllIDs: function () {
+ let items = {};
+ let logins = Services.logins.getAllLogins({});
+
+ for (let i = 0; i < logins.length; i++) {
+ // Skip over Weave password/passphrase entries.
+ let metaInfo = logins[i].QueryInterface(Ci.nsILoginMetaInfo);
+ if (Utils.getSyncCredentialsHosts().has(metaInfo.hostname)) {
+ continue;
+ }
+
+ items[metaInfo.guid] = metaInfo;
+ }
+
+ return items;
+ },
+
+ changeItemID: function (oldID, newID) {
+ this._log.trace("Changing item ID: " + oldID + " to " + newID);
+
+ let oldLogin = this._getLoginFromGUID(oldID);
+ if (!oldLogin) {
+ this._log.trace("Can't change item ID: item doesn't exist");
+ return;
+ }
+ if (this._getLoginFromGUID(newID)) {
+ this._log.trace("Can't change item ID: new ID already in use");
+ return;
+ }
+
+ let prop = this._newPropertyBag();
+ prop.setPropertyAsAUTF8String("guid", newID);
+
+ Services.logins.modifyLogin(oldLogin, prop);
+ },
+
+ itemExists: function (id) {
+ return !!this._getLoginFromGUID(id);
+ },
+
+ createRecord: function (id, collection) {
+ let record = new LoginRec(collection, id);
+ let login = this._getLoginFromGUID(id);
+
+ if (!login) {
+ record.deleted = true;
+ return record;
+ }
+
+ record.hostname = login.hostname;
+ record.formSubmitURL = login.formSubmitURL;
+ record.httpRealm = login.httpRealm;
+ record.username = login.username;
+ record.password = login.password;
+ record.usernameField = login.usernameField;
+ record.passwordField = login.passwordField;
+
+ // Optional fields.
+ login.QueryInterface(Ci.nsILoginMetaInfo);
+ record.timeCreated = login.timeCreated;
+ record.timePasswordChanged = login.timePasswordChanged;
+
+ return record;
+ },
+
+ create: function (record) {
+ let login = this._nsLoginInfoFromRecord(record);
+ if (!login) {
+ return;
+ }
+
+ this._log.debug("Adding login for " + record.hostname);
+ this._log.trace("httpRealm: " + JSON.stringify(login.httpRealm) + "; " +
+ "formSubmitURL: " + JSON.stringify(login.formSubmitURL));
+ try {
+ Services.logins.addLogin(login);
+ } catch(ex) {
+ this._log.debug(`Adding record ${record.id} resulted in exception`, ex);
+ }
+ },
+
+ remove: function (record) {
+ this._log.trace("Removing login " + record.id);
+
+ let loginItem = this._getLoginFromGUID(record.id);
+ if (!loginItem) {
+ this._log.trace("Asked to remove record that doesn't exist, ignoring");
+ return;
+ }
+
+ Services.logins.removeLogin(loginItem);
+ },
+
+ update: function (record) {
+ let loginItem = this._getLoginFromGUID(record.id);
+ if (!loginItem) {
+ this._log.debug("Skipping update for unknown item: " + record.hostname);
+ return;
+ }
+
+ this._log.debug("Updating " + record.hostname);
+ let newinfo = this._nsLoginInfoFromRecord(record);
+ if (!newinfo) {
+ return;
+ }
+
+ try {
+ Services.logins.modifyLogin(loginItem, newinfo);
+ } catch(ex) {
+ this._log.debug(`Modifying record ${record.id} resulted in exception; not modifying`, ex);
+ }
+ },
+
+ wipe: function () {
+ Services.logins.removeAllLogins();
+ },
+};
+
+function PasswordTracker(name, engine) {
+ Tracker.call(this, name, engine);
+ Svc.Obs.add("weave:engine:start-tracking", this);
+ Svc.Obs.add("weave:engine:stop-tracking", this);
+}
+PasswordTracker.prototype = {
+ __proto__: Tracker.prototype,
+
+ startTracking: function () {
+ Svc.Obs.add("passwordmgr-storage-changed", this);
+ },
+
+ stopTracking: function () {
+ Svc.Obs.remove("passwordmgr-storage-changed", this);
+ },
+
+ observe: function (subject, topic, data) {
+ Tracker.prototype.observe.call(this, subject, topic, data);
+
+ if (this.ignoreAll) {
+ return;
+ }
+
+ // A single add, remove or change or removing all items
+ // will trigger a sync for MULTI_DEVICE.
+ switch (data) {
+ case "modifyLogin":
+ subject = subject.QueryInterface(Ci.nsIArray).queryElementAt(1, Ci.nsILoginMetaInfo);
+ // Fall through.
+ case "addLogin":
+ case "removeLogin":
+ // Skip over Weave password/passphrase changes.
+ subject.QueryInterface(Ci.nsILoginMetaInfo).QueryInterface(Ci.nsILoginInfo);
+ if (Utils.getSyncCredentialsHosts().has(subject.hostname)) {
+ break;
+ }
+
+ this.score += SCORE_INCREMENT_XLARGE;
+ this._log.trace(data + ": " + subject.guid);
+ this.addChangedID(subject.guid);
+ break;
+ case "removeAllLogins":
+ this._log.trace(data);
+ this.score += SCORE_INCREMENT_XLARGE;
+ break;
+ }
+ },
+};
+
+class PasswordValidator extends CollectionValidator {
+ constructor() {
+ super("passwords", "id", [
+ "hostname",
+ "formSubmitURL",
+ "httpRealm",
+ "password",
+ "passwordField",
+ "username",
+ "usernameField",
+ ]);
+ }
+
+ getClientItems() {
+ let logins = Services.logins.getAllLogins({});
+ let syncHosts = Utils.getSyncCredentialsHosts()
+ let result = logins.map(l => l.QueryInterface(Ci.nsILoginMetaInfo))
+ .filter(l => !syncHosts.has(l.hostname));
+ return Promise.resolve(result);
+ }
+
+ normalizeClientItem(item) {
+ return {
+ id: item.guid,
+ guid: item.guid,
+ hostname: item.hostname,
+ formSubmitURL: item.formSubmitURL,
+ httpRealm: item.httpRealm,
+ password: item.password,
+ passwordField: item.passwordField,
+ username: item.username,
+ usernameField: item.usernameField,
+ original: item,
+ }
+ }
+
+ normalizeServerItem(item) {
+ return Object.assign({ guid: item.id }, item);
+ }
+}
+
+
diff --git a/services/sync/modules/engines/prefs.js b/services/sync/modules/engines/prefs.js
new file mode 100644
index 000000000..9ceeb9ac6
--- /dev/null
+++ b/services/sync/modules/engines/prefs.js
@@ -0,0 +1,273 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ['PrefsEngine', 'PrefRec'];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cu = Components.utils;
+
+const PREF_SYNC_PREFS_PREFIX = "services.sync.prefs.sync.";
+
+Cu.import("resource://services-sync/engines.js");
+Cu.import("resource://services-sync/record.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://services-common/utils.js");
+Cu.import("resource://gre/modules/LightweightThemeManager.jsm");
+Cu.import("resource://gre/modules/Preferences.jsm");
+
+const PREFS_GUID = CommonUtils.encodeBase64URL(Services.appinfo.ID);
+
+this.PrefRec = function PrefRec(collection, id) {
+ CryptoWrapper.call(this, collection, id);
+}
+PrefRec.prototype = {
+ __proto__: CryptoWrapper.prototype,
+ _logName: "Sync.Record.Pref",
+};
+
+Utils.deferGetSet(PrefRec, "cleartext", ["value"]);
+
+
+this.PrefsEngine = function PrefsEngine(service) {
+ SyncEngine.call(this, "Prefs", service);
+}
+PrefsEngine.prototype = {
+ __proto__: SyncEngine.prototype,
+ _storeObj: PrefStore,
+ _trackerObj: PrefTracker,
+ _recordObj: PrefRec,
+ version: 2,
+
+ syncPriority: 1,
+ allowSkippedRecord: false,
+
+ getChangedIDs: function () {
+ // No need for a proper timestamp (no conflict resolution needed).
+ let changedIDs = {};
+ if (this._tracker.modified)
+ changedIDs[PREFS_GUID] = 0;
+ return changedIDs;
+ },
+
+ _wipeClient: function () {
+ SyncEngine.prototype._wipeClient.call(this);
+ this.justWiped = true;
+ },
+
+ _reconcile: function (item) {
+ // Apply the incoming item if we don't care about the local data
+ if (this.justWiped) {
+ this.justWiped = false;
+ return true;
+ }
+ return SyncEngine.prototype._reconcile.call(this, item);
+ }
+};
+
+
+function PrefStore(name, engine) {
+ Store.call(this, name, engine);
+ Svc.Obs.add("profile-before-change", function () {
+ this.__prefs = null;
+ }, this);
+}
+PrefStore.prototype = {
+ __proto__: Store.prototype,
+
+ __prefs: null,
+ get _prefs() {
+ if (!this.__prefs) {
+ this.__prefs = new Preferences();
+ }
+ return this.__prefs;
+ },
+
+ _getSyncPrefs: function () {
+ let syncPrefs = Cc["@mozilla.org/preferences-service;1"]
+ .getService(Ci.nsIPrefService)
+ .getBranch(PREF_SYNC_PREFS_PREFIX)
+ .getChildList("", {});
+ // Also sync preferences that determine which prefs get synced.
+ let controlPrefs = syncPrefs.map(pref => PREF_SYNC_PREFS_PREFIX + pref);
+ return controlPrefs.concat(syncPrefs);
+ },
+
+ _isSynced: function (pref) {
+ return pref.startsWith(PREF_SYNC_PREFS_PREFIX) ||
+ this._prefs.get(PREF_SYNC_PREFS_PREFIX + pref, false);
+ },
+
+ _getAllPrefs: function () {
+ let values = {};
+ for (let pref of this._getSyncPrefs()) {
+ if (this._isSynced(pref)) {
+ // Missing and default prefs get the null value.
+ values[pref] = this._prefs.isSet(pref) ? this._prefs.get(pref, null) : null;
+ }
+ }
+ return values;
+ },
+
+ _updateLightWeightTheme (themeID) {
+ let themeObject = null;
+ if (themeID) {
+ themeObject = LightweightThemeManager.getUsedTheme(themeID);
+ }
+ LightweightThemeManager.currentTheme = themeObject;
+ },
+
+ _setAllPrefs: function (values) {
+ let selectedThemeIDPref = "lightweightThemes.selectedThemeID";
+ let selectedThemeIDBefore = this._prefs.get(selectedThemeIDPref, null);
+ let selectedThemeIDAfter = selectedThemeIDBefore;
+
+ // Update 'services.sync.prefs.sync.foo.pref' before 'foo.pref', otherwise
+ // _isSynced returns false when 'foo.pref' doesn't exist (e.g., on a new device).
+ let prefs = Object.keys(values).sort(a => -a.indexOf(PREF_SYNC_PREFS_PREFIX));
+ for (let pref of prefs) {
+ if (!this._isSynced(pref)) {
+ continue;
+ }
+
+ let value = values[pref];
+
+ switch (pref) {
+ // Some special prefs we don't want to set directly.
+ case selectedThemeIDPref:
+ selectedThemeIDAfter = value;
+ break;
+
+ // default is to just set the pref
+ default:
+ if (value == null) {
+ // Pref has gone missing. The best we can do is reset it.
+ this._prefs.reset(pref);
+ } else {
+ try {
+ this._prefs.set(pref, value);
+ } catch(ex) {
+ this._log.trace("Failed to set pref: " + pref + ": " + ex);
+ }
+ }
+ }
+ }
+
+ // Notify the lightweight theme manager if the selected theme has changed.
+ if (selectedThemeIDBefore != selectedThemeIDAfter) {
+ this._updateLightWeightTheme(selectedThemeIDAfter);
+ }
+ },
+
+ getAllIDs: function () {
+ /* We store all prefs in just one WBO, with just one GUID */
+ let allprefs = {};
+ allprefs[PREFS_GUID] = true;
+ return allprefs;
+ },
+
+ changeItemID: function (oldID, newID) {
+ this._log.trace("PrefStore GUID is constant!");
+ },
+
+ itemExists: function (id) {
+ return (id === PREFS_GUID);
+ },
+
+ createRecord: function (id, collection) {
+ let record = new PrefRec(collection, id);
+
+ if (id == PREFS_GUID) {
+ record.value = this._getAllPrefs();
+ } else {
+ record.deleted = true;
+ }
+
+ return record;
+ },
+
+ create: function (record) {
+ this._log.trace("Ignoring create request");
+ },
+
+ remove: function (record) {
+ this._log.trace("Ignoring remove request");
+ },
+
+ update: function (record) {
+ // Silently ignore pref updates that are for other apps.
+ if (record.id != PREFS_GUID)
+ return;
+
+ this._log.trace("Received pref updates, applying...");
+ this._setAllPrefs(record.value);
+ },
+
+ wipe: function () {
+ this._log.trace("Ignoring wipe request");
+ }
+};
+
+function PrefTracker(name, engine) {
+ Tracker.call(this, name, engine);
+ Svc.Obs.add("profile-before-change", this);
+ Svc.Obs.add("weave:engine:start-tracking", this);
+ Svc.Obs.add("weave:engine:stop-tracking", this);
+}
+PrefTracker.prototype = {
+ __proto__: Tracker.prototype,
+
+ get modified() {
+ return Svc.Prefs.get("engine.prefs.modified", false);
+ },
+ set modified(value) {
+ Svc.Prefs.set("engine.prefs.modified", value);
+ },
+
+ loadChangedIDs: function loadChangedIDs() {
+ // Don't read changed IDs from disk at start up.
+ },
+
+ clearChangedIDs: function clearChangedIDs() {
+ this.modified = false;
+ },
+
+ __prefs: null,
+ get _prefs() {
+ if (!this.__prefs) {
+ this.__prefs = new Preferences();
+ }
+ return this.__prefs;
+ },
+
+ startTracking: function () {
+ Services.prefs.addObserver("", this, false);
+ },
+
+ stopTracking: function () {
+ this.__prefs = null;
+ Services.prefs.removeObserver("", this);
+ },
+
+ observe: function (subject, topic, data) {
+ Tracker.prototype.observe.call(this, subject, topic, data);
+
+ switch (topic) {
+ case "profile-before-change":
+ this.stopTracking();
+ break;
+ case "nsPref:changed":
+ // Trigger a sync for MULTI-DEVICE for a change that determines
+ // which prefs are synced or a regular pref change.
+ if (data.indexOf(PREF_SYNC_PREFS_PREFIX) == 0 ||
+ this._prefs.get(PREF_SYNC_PREFS_PREFIX + data, false)) {
+ this.score += SCORE_INCREMENT_XLARGE;
+ this.modified = true;
+ this._log.trace("Preference " + data + " changed");
+ }
+ break;
+ }
+ }
+};
diff --git a/services/sync/modules/engines/tabs.js b/services/sync/modules/engines/tabs.js
new file mode 100644
index 000000000..45ece4a23
--- /dev/null
+++ b/services/sync/modules/engines/tabs.js
@@ -0,0 +1,393 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = ["TabEngine", "TabSetRecord"];
+
+var {classes: Cc, interfaces: Ci, utils: Cu} = Components;
+
+const TABS_TTL = 604800; // 7 days.
+const TAB_ENTRIES_LIMIT = 25; // How many URLs to include in tab history.
+
+Cu.import("resource://gre/modules/Preferences.jsm");
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+Cu.import("resource://services-sync/engines.js");
+Cu.import("resource://services-sync/engines/clients.js");
+Cu.import("resource://services-sync/record.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://services-sync/constants.js");
+
+XPCOMUtils.defineLazyModuleGetter(this, "PrivateBrowsingUtils",
+ "resource://gre/modules/PrivateBrowsingUtils.jsm");
+
+this.TabSetRecord = function TabSetRecord(collection, id) {
+ CryptoWrapper.call(this, collection, id);
+}
+TabSetRecord.prototype = {
+ __proto__: CryptoWrapper.prototype,
+ _logName: "Sync.Record.Tabs",
+ ttl: TABS_TTL,
+};
+
+Utils.deferGetSet(TabSetRecord, "cleartext", ["clientName", "tabs"]);
+
+
+this.TabEngine = function TabEngine(service) {
+ SyncEngine.call(this, "Tabs", service);
+
+ // Reset the client on every startup so that we fetch recent tabs.
+ this._resetClient();
+}
+TabEngine.prototype = {
+ __proto__: SyncEngine.prototype,
+ _storeObj: TabStore,
+ _trackerObj: TabTracker,
+ _recordObj: TabSetRecord,
+ // A flag to indicate if we have synced in this session. This is to help
+ // consumers of remote tabs that may want to differentiate between "I've an
+ // empty tab list as I haven't yet synced" vs "I've an empty tab list
+ // as there really are no tabs"
+ hasSyncedThisSession: false,
+
+ syncPriority: 3,
+
+ getChangedIDs: function () {
+ // No need for a proper timestamp (no conflict resolution needed).
+ let changedIDs = {};
+ if (this._tracker.modified)
+ changedIDs[this.service.clientsEngine.localID] = 0;
+ return changedIDs;
+ },
+
+ // API for use by Sync UI code to give user choices of tabs to open.
+ getAllClients: function () {
+ return this._store._remoteClients;
+ },
+
+ getClientById: function (id) {
+ return this._store._remoteClients[id];
+ },
+
+ _resetClient: function () {
+ SyncEngine.prototype._resetClient.call(this);
+ this._store.wipe();
+ this._tracker.modified = true;
+ this.hasSyncedThisSession = false;
+ },
+
+ removeClientData: function () {
+ let url = this.engineURL + "/" + this.service.clientsEngine.localID;
+ this.service.resource(url).delete();
+ },
+
+ /**
+ * Return a Set of open URLs.
+ */
+ getOpenURLs: function () {
+ let urls = new Set();
+ for (let entry of this._store.getAllTabs()) {
+ urls.add(entry.urlHistory[0]);
+ }
+ return urls;
+ },
+
+ _reconcile: function (item) {
+ // Skip our own record.
+ // TabStore.itemExists tests only against our local client ID.
+ if (this._store.itemExists(item.id)) {
+ this._log.trace("Ignoring incoming tab item because of its id: " + item.id);
+ return false;
+ }
+
+ return SyncEngine.prototype._reconcile.call(this, item);
+ },
+
+ _syncFinish() {
+ this.hasSyncedThisSession = true;
+ return SyncEngine.prototype._syncFinish.call(this);
+ },
+};
+
+
+function TabStore(name, engine) {
+ Store.call(this, name, engine);
+}
+TabStore.prototype = {
+ __proto__: Store.prototype,
+
+ itemExists: function (id) {
+ return id == this.engine.service.clientsEngine.localID;
+ },
+
+ getWindowEnumerator: function () {
+ return Services.wm.getEnumerator("navigator:browser");
+ },
+
+ shouldSkipWindow: function (win) {
+ return win.closed ||
+ PrivateBrowsingUtils.isWindowPrivate(win);
+ },
+
+ getTabState: function (tab) {
+ return JSON.parse(Svc.Session.getTabState(tab));
+ },
+
+ getAllTabs: function (filter) {
+ let filteredUrls = new RegExp(Svc.Prefs.get("engine.tabs.filteredUrls"), "i");
+
+ let allTabs = [];
+
+ let winEnum = this.getWindowEnumerator();
+ while (winEnum.hasMoreElements()) {
+ let win = winEnum.getNext();
+ if (this.shouldSkipWindow(win)) {
+ continue;
+ }
+
+ for (let tab of win.gBrowser.tabs) {
+ let tabState = this.getTabState(tab);
+
+ // Make sure there are history entries to look at.
+ if (!tabState || !tabState.entries.length) {
+ continue;
+ }
+
+ let acceptable = !filter ? (url) => url :
+ (url) => url && !filteredUrls.test(url);
+
+ let entries = tabState.entries;
+ let index = tabState.index;
+ let current = entries[index - 1];
+
+ // We ignore the tab completely if the current entry url is
+ // not acceptable (we need something accurate to open).
+ if (!acceptable(current.url)) {
+ continue;
+ }
+
+ if (current.url.length >= (MAX_UPLOAD_BYTES - 1000)) {
+ this._log.trace("Skipping over-long URL.");
+ continue;
+ }
+
+ // The element at `index` is the current page. Previous URLs were
+ // previously visited URLs; subsequent URLs are in the 'forward' stack,
+ // which we can't represent in Sync, so we truncate here.
+ let candidates = (entries.length == index) ?
+ entries :
+ entries.slice(0, index);
+
+ let urls = candidates.map((entry) => entry.url)
+ .filter(acceptable)
+ .reverse(); // Because Sync puts current at index 0, and history after.
+
+ // Truncate if necessary.
+ if (urls.length > TAB_ENTRIES_LIMIT) {
+ urls.length = TAB_ENTRIES_LIMIT;
+ }
+
+ allTabs.push({
+ title: current.title || "",
+ urlHistory: urls,
+ icon: tabState.image ||
+ (tabState.attributes && tabState.attributes.image) ||
+ "",
+ lastUsed: Math.floor((tabState.lastAccessed || 0) / 1000),
+ });
+ }
+ }
+
+ return allTabs;
+ },
+
+ createRecord: function (id, collection) {
+ let record = new TabSetRecord(collection, id);
+ record.clientName = this.engine.service.clientsEngine.localName;
+
+ // Sort tabs in descending-used order to grab the most recently used
+ let tabs = this.getAllTabs(true).sort(function (a, b) {
+ return b.lastUsed - a.lastUsed;
+ });
+
+ // Figure out how many tabs we can pack into a payload. Starting with a 28KB
+ // payload, we can estimate various overheads from encryption/JSON/WBO.
+ let size = JSON.stringify(tabs).length;
+ let origLength = tabs.length;
+ const MAX_TAB_SIZE = 20000;
+ if (size > MAX_TAB_SIZE) {
+ // Estimate a little more than the direct fraction to maximize packing
+ let cutoff = Math.ceil(tabs.length * MAX_TAB_SIZE / size);
+ tabs = tabs.slice(0, cutoff + 1);
+
+ // Keep dropping off the last entry until the data fits
+ while (JSON.stringify(tabs).length > MAX_TAB_SIZE)
+ tabs.pop();
+ }
+
+ this._log.trace("Created tabs " + tabs.length + " of " + origLength);
+ tabs.forEach(function (tab) {
+ this._log.trace("Wrapping tab: " + JSON.stringify(tab));
+ }, this);
+
+ record.tabs = tabs;
+ return record;
+ },
+
+ getAllIDs: function () {
+ // Don't report any tabs if all windows are in private browsing for
+ // first syncs.
+ let ids = {};
+ let allWindowsArePrivate = false;
+ let wins = Services.wm.getEnumerator("navigator:browser");
+ while (wins.hasMoreElements()) {
+ if (PrivateBrowsingUtils.isWindowPrivate(wins.getNext())) {
+ // Ensure that at least there is a private window.
+ allWindowsArePrivate = true;
+ } else {
+ // If there is a not private windown then finish and continue.
+ allWindowsArePrivate = false;
+ break;
+ }
+ }
+
+ if (allWindowsArePrivate &&
+ !PrivateBrowsingUtils.permanentPrivateBrowsing) {
+ return ids;
+ }
+
+ ids[this.engine.service.clientsEngine.localID] = true;
+ return ids;
+ },
+
+ wipe: function () {
+ this._remoteClients = {};
+ },
+
+ create: function (record) {
+ this._log.debug("Adding remote tabs from " + record.clientName);
+ this._remoteClients[record.id] = Object.assign({}, record.cleartext, {
+ lastModified: record.modified
+ });
+ },
+
+ update: function (record) {
+ this._log.trace("Ignoring tab updates as local ones win");
+ },
+};
+
+
+function TabTracker(name, engine) {
+ Tracker.call(this, name, engine);
+ Svc.Obs.add("weave:engine:start-tracking", this);
+ Svc.Obs.add("weave:engine:stop-tracking", this);
+
+ // Make sure "this" pointer is always set correctly for event listeners.
+ this.onTab = Utils.bind2(this, this.onTab);
+ this._unregisterListeners = Utils.bind2(this, this._unregisterListeners);
+}
+TabTracker.prototype = {
+ __proto__: Tracker.prototype,
+
+ QueryInterface: XPCOMUtils.generateQI([Ci.nsIObserver]),
+
+ loadChangedIDs: function () {
+ // Don't read changed IDs from disk at start up.
+ },
+
+ clearChangedIDs: function () {
+ this.modified = false;
+ },
+
+ _topics: ["pageshow", "TabOpen", "TabClose", "TabSelect"],
+
+ _registerListenersForWindow: function (window) {
+ this._log.trace("Registering tab listeners in window");
+ for (let topic of this._topics) {
+ window.addEventListener(topic, this.onTab, false);
+ }
+ window.addEventListener("unload", this._unregisterListeners, false);
+ // If it's got a tab browser we can listen for things like navigation.
+ if (window.gBrowser) {
+ window.gBrowser.addProgressListener(this);
+ }
+ },
+
+ _unregisterListeners: function (event) {
+ this._unregisterListenersForWindow(event.target);
+ },
+
+ _unregisterListenersForWindow: function (window) {
+ this._log.trace("Removing tab listeners in window");
+ window.removeEventListener("unload", this._unregisterListeners, false);
+ for (let topic of this._topics) {
+ window.removeEventListener(topic, this.onTab, false);
+ }
+ if (window.gBrowser) {
+ window.gBrowser.removeProgressListener(this);
+ }
+ },
+
+ startTracking: function () {
+ Svc.Obs.add("domwindowopened", this);
+ let wins = Services.wm.getEnumerator("navigator:browser");
+ while (wins.hasMoreElements()) {
+ this._registerListenersForWindow(wins.getNext());
+ }
+ },
+
+ stopTracking: function () {
+ Svc.Obs.remove("domwindowopened", this);
+ let wins = Services.wm.getEnumerator("navigator:browser");
+ while (wins.hasMoreElements()) {
+ this._unregisterListenersForWindow(wins.getNext());
+ }
+ },
+
+ observe: function (subject, topic, data) {
+ Tracker.prototype.observe.call(this, subject, topic, data);
+
+ switch (topic) {
+ case "domwindowopened":
+ let onLoad = () => {
+ subject.removeEventListener("load", onLoad, false);
+ // Only register after the window is done loading to avoid unloads.
+ this._registerListenersForWindow(subject);
+ };
+
+ // Add tab listeners now that a window has opened.
+ subject.addEventListener("load", onLoad, false);
+ break;
+ }
+ },
+
+ onTab: function (event) {
+ if (event.originalTarget.linkedBrowser) {
+ let browser = event.originalTarget.linkedBrowser;
+ if (PrivateBrowsingUtils.isBrowserPrivate(browser) &&
+ !PrivateBrowsingUtils.permanentPrivateBrowsing) {
+ this._log.trace("Ignoring tab event from private browsing.");
+ return;
+ }
+ }
+
+ this._log.trace("onTab event: " + event.type);
+ this.modified = true;
+
+ // For page shows, bump the score 10% of the time, emulating a partial
+ // score. We don't want to sync too frequently. For all other page
+ // events, always bump the score.
+ if (event.type != "pageshow" || Math.random() < .1) {
+ this.score += SCORE_INCREMENT_SMALL;
+ }
+ },
+
+ // web progress listeners.
+ onLocationChange: function (webProgress, request, location, flags) {
+ // We only care about top-level location changes which are not in the same
+ // document.
+ if (webProgress.isTopLevel &&
+ ((flags & Ci.nsIWebProgressListener.LOCATION_CHANGE_SAME_DOCUMENT) == 0)) {
+ this.modified = true;
+ }
+ },
+};