summaryrefslogtreecommitdiffstats
path: root/services/sync/modules
diff options
context:
space:
mode:
Diffstat (limited to 'services/sync/modules')
-rw-r--r--services/sync/modules/FxaMigrator.jsm546
-rw-r--r--services/sync/modules/SyncedTabs.jsm301
-rw-r--r--services/sync/modules/addonsreconciler.js35
-rw-r--r--services/sync/modules/addonutils.js94
-rw-r--r--services/sync/modules/bookmark_validator.js784
-rw-r--r--services/sync/modules/browserid_identity.js151
-rw-r--r--services/sync/modules/collection_validator.js204
-rw-r--r--services/sync/modules/constants.js31
-rw-r--r--services/sync/modules/engines.js474
-rw-r--r--services/sync/modules/engines/addons.js140
-rw-r--r--services/sync/modules/engines/bookmarks.js1771
-rw-r--r--services/sync/modules/engines/clients.js461
-rw-r--r--services/sync/modules/engines/forms.js102
-rw-r--r--services/sync/modules/engines/history.js68
-rw-r--r--services/sync/modules/engines/passwords.js89
-rw-r--r--services/sync/modules/engines/prefs.js81
-rw-r--r--services/sync/modules/engines/tabs.js71
-rw-r--r--services/sync/modules/healthreport.jsm262
-rw-r--r--services/sync/modules/identity.js43
-rw-r--r--services/sync/modules/jpakeclient.js8
-rw-r--r--services/sync/modules/keys.js2
-rw-r--r--services/sync/modules/main.js16
-rw-r--r--services/sync/modules/notifications.js8
-rw-r--r--services/sync/modules/policies.js211
-rw-r--r--services/sync/modules/record.js481
-rw-r--r--services/sync/modules/resource.js88
-rw-r--r--services/sync/modules/rest.js20
-rw-r--r--services/sync/modules/service.js283
-rw-r--r--services/sync/modules/stages/cluster.js6
-rw-r--r--services/sync/modules/stages/declined.js6
-rw-r--r--services/sync/modules/stages/enginesync.js147
-rw-r--r--services/sync/modules/status.js13
-rw-r--r--services/sync/modules/telemetry.js578
-rw-r--r--services/sync/modules/userapi.js2
-rw-r--r--services/sync/modules/util.js148
35 files changed, 2602 insertions, 5123 deletions
diff --git a/services/sync/modules/FxaMigrator.jsm b/services/sync/modules/FxaMigrator.jsm
new file mode 100644
index 000000000..605ee5d7f
--- /dev/null
+++ b/services/sync/modules/FxaMigrator.jsm
@@ -0,0 +1,546 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict;"
+
+const {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components;
+
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://gre/modules/Task.jsm");
+
+XPCOMUtils.defineLazyModuleGetter(this, "fxAccounts",
+ "resource://gre/modules/FxAccounts.jsm");
+
+XPCOMUtils.defineLazyGetter(this, "WeaveService", function() {
+ return Cc["@mozilla.org/weave/service;1"]
+ .getService(Components.interfaces.nsISupports)
+ .wrappedJSObject;
+});
+
+XPCOMUtils.defineLazyModuleGetter(this, "Weave",
+ "resource://services-sync/main.js");
+
+// FxAccountsCommon.js doesn't use a "namespace", so create one here.
+let fxAccountsCommon = {};
+Cu.import("resource://gre/modules/FxAccountsCommon.js", fxAccountsCommon);
+
+// We send this notification whenever the "user" migration state changes.
+const OBSERVER_STATE_CHANGE_TOPIC = "fxa-migration:state-changed";
+// We also send the state notification when we *receive* this. This allows
+// consumers to avoid loading this module until it receives a notification
+// from us (which may never happen if there's no migration to do)
+const OBSERVER_STATE_REQUEST_TOPIC = "fxa-migration:state-request";
+
+// We send this notification whenever the migration is paused waiting for
+// something internal to complete.
+const OBSERVER_INTERNAL_STATE_CHANGE_TOPIC = "fxa-migration:internal-state-changed";
+
+// We use this notification so Sync's healthreport module can record telemetry
+// (actually via "health report") for us.
+const OBSERVER_INTERNAL_TELEMETRY_TOPIC = "fxa-migration:internal-telemetry";
+
+const OBSERVER_TOPICS = [
+ "xpcom-shutdown",
+ "weave:service:sync:start",
+ "weave:service:sync:finish",
+ "weave:service:sync:error",
+ "weave:eol",
+ OBSERVER_STATE_REQUEST_TOPIC,
+ fxAccountsCommon.ONLOGIN_NOTIFICATION,
+ fxAccountsCommon.ONLOGOUT_NOTIFICATION,
+ fxAccountsCommon.ONVERIFIED_NOTIFICATION,
+];
+
+// A list of preference names we write to the migration sentinel. We only
+// write ones that have a user-set value.
+const FXA_SENTINEL_PREFS = [
+ "identity.fxaccounts.auth.uri",
+ "identity.fxaccounts.remote.force_auth.uri",
+ "identity.fxaccounts.remote.signup.uri",
+ "identity.fxaccounts.remote.signin.uri",
+ "identity.fxaccounts.settings.uri",
+ "services.sync.tokenServerURI",
+];
+
+function Migrator() {
+ // Leave the log-level as Debug - Sync will setup log appenders such that
+ // these messages generally will not be seen unless other log related
+ // prefs are set.
+ this.log.level = Log.Level.Debug;
+
+ this._nextUserStatePromise = Promise.resolve();
+
+ for (let topic of OBSERVER_TOPICS) {
+ Services.obs.addObserver(this, topic, false);
+ }
+ // ._state is an optimization so we avoid sending redundant observer
+ // notifications when the state hasn't actually changed.
+ this._state = null;
+}
+
+Migrator.prototype = {
+ log: Log.repository.getLogger("Sync.SyncMigration"),
+
+ // What user action is necessary to push the migration forward?
+ // A |null| state means there is nothing to do. Note that a null state implies
+ // either. (a) no migration is necessary or (b) that the migrator module is
+ // waiting for something outside of the user's control - eg, sync to complete,
+ // the migration sentinel to be uploaded, etc. In most cases the wait will be
+ // short, but edge cases (eg, no network, sync bugs that prevent it stopping
+ // until shutdown) may require a significantly longer wait.
+ STATE_USER_FXA: "waiting for user to be signed in to FxA",
+ STATE_USER_FXA_VERIFIED: "waiting for a verified FxA user",
+
+ // What internal state are we at? This is primarily used for FHR reporting so
+ // we can determine why exactly we might be stalled.
+ STATE_INTERNAL_WAITING_SYNC_COMPLETE: "waiting for sync to complete",
+ STATE_INTERNAL_WAITING_WRITE_SENTINEL: "waiting for sentinel to be written",
+ STATE_INTERNAL_WAITING_START_OVER: "waiting for sync to reset itself",
+ STATE_INTERNAL_COMPLETE: "migration complete",
+
+ // Flags for the telemetry we record. The UI will call a helper to record
+ // the fact some UI was interacted with.
+ TELEMETRY_ACCEPTED: "accepted",
+ TELEMETRY_DECLINED: "declined",
+ TELEMETRY_UNLINKED: "unlinked",
+
+ finalize() {
+ for (let topic of OBSERVER_TOPICS) {
+ Services.obs.removeObserver(this, topic);
+ }
+ },
+
+ observe(subject, topic, data) {
+ this.log.debug("observed " + topic);
+ switch (topic) {
+ case "xpcom-shutdown":
+ this.finalize();
+ break;
+
+ case OBSERVER_STATE_REQUEST_TOPIC:
+ // someone has requested the state - send it.
+ this._queueCurrentUserState(true);
+ break;
+
+ default:
+ // some other observer that may affect our state has fired, so update.
+ this._queueCurrentUserState().then(
+ () => this.log.debug("update state from observer " + topic + " complete")
+ ).catch(err => {
+ let msg = "Failed to handle topic " + topic + ": " + err;
+ Cu.reportError(msg);
+ this.log.error(msg);
+ });
+ }
+ },
+
+ // Try and move to a state where we are blocked on a user action.
+ // This needs to be restartable, and the states may, in edge-cases, end
+ // up going backwards (eg, user logs out while we are waiting to be told
+ // about verification)
+ // This is called by our observer notifications - so if there is already
+ // a promise in-flight, it's possible we will miss something important - so
+ // we wait for the in-flight one to complete then fire another (ie, this
+ // is effectively a queue of promises)
+ _queueCurrentUserState(forceObserver = false) {
+ return this._nextUserStatePromise = this._nextUserStatePromise.then(
+ () => this._promiseCurrentUserState(forceObserver),
+ err => {
+ let msg = "Failed to determine the current user state: " + err;
+ Cu.reportError(msg);
+ this.log.error(msg);
+ return this._promiseCurrentUserState(forceObserver)
+ }
+ );
+ },
+
+ _promiseCurrentUserState: Task.async(function* (forceObserver) {
+ this.log.trace("starting _promiseCurrentUserState");
+ let update = (newState, email=null) => {
+ this.log.info("Migration state: '${state}' => '${newState}'",
+ {state: this._state, newState: newState});
+ if (forceObserver || newState !== this._state) {
+ this._state = newState;
+ let subject = Cc["@mozilla.org/supports-string;1"]
+ .createInstance(Ci.nsISupportsString);
+ subject.data = email || "";
+ Services.obs.notifyObservers(subject, OBSERVER_STATE_CHANGE_TOPIC, newState);
+ }
+ return newState;
+ }
+
+ // If we have no sync user, or are already using an FxA account we must
+ // be done.
+ if (WeaveService.fxAccountsEnabled) {
+ // should not be necessary, but if we somehow ended up with FxA enabled
+ // and sync blocked it would be bad - so better safe than sorry.
+ this.log.debug("FxA enabled - there's nothing to do!")
+ this._unblockSync();
+ return update(null);
+ }
+
+ // so we need to migrate - let's see how far along we are.
+ // If sync isn't in EOL mode, then we are still waiting for the server
+ // to offer the migration process - so no user action necessary.
+ let isEOL = false;
+ try {
+ isEOL = !!Services.prefs.getCharPref("services.sync.errorhandler.alert.mode");
+ } catch (e) {}
+
+ if (!isEOL) {
+ return update(null);
+ }
+
+ // So we are in EOL mode - have we a user?
+ let fxauser = yield fxAccounts.getSignedInUser();
+ if (!fxauser) {
+ // See if there is a migration sentinel so we can send the email
+ // address that was used on a different device for this account (ie, if
+ // this is a "join the party" migration rather than the first)
+ let sentinel = yield this._getSyncMigrationSentinel();
+ return update(this.STATE_USER_FXA, sentinel && sentinel.email);
+ }
+ if (!fxauser.verified) {
+ return update(this.STATE_USER_FXA_VERIFIED, fxauser.email);
+ }
+
+ // So we just have housekeeping to do - we aren't blocked on a user, so
+ // reflect that.
+ this.log.info("No next user state - doing some housekeeping");
+ update(null);
+
+ // We need to disable sync from automatically starting,
+ // and if we are currently syncing wait for it to complete.
+ this._blockSync();
+
+ // Are we currently syncing?
+ if (Weave.Service._locked) {
+ // our observers will kick us further along when complete.
+ this.log.info("waiting for sync to complete")
+ Services.obs.notifyObservers(null, OBSERVER_INTERNAL_STATE_CHANGE_TOPIC,
+ this.STATE_INTERNAL_WAITING_SYNC_COMPLETE);
+ return null;
+ }
+
+ // Write the migration sentinel if necessary.
+ Services.obs.notifyObservers(null, OBSERVER_INTERNAL_STATE_CHANGE_TOPIC,
+ this.STATE_INTERNAL_WAITING_WRITE_SENTINEL);
+ yield this._setMigrationSentinelIfNecessary();
+
+ // Get the list of enabled engines to we can restore that state.
+ let enginePrefs = this._getEngineEnabledPrefs();
+
+ // Must be ready to perform the actual migration.
+ this.log.info("Performing final sync migration steps");
+ // Do the actual migration. We setup one observer for when the new identity
+ // is about to be initialized so we can reset some key preferences - but
+ // there's no promise associated with this.
+ let observeStartOverIdentity;
+ Services.obs.addObserver(observeStartOverIdentity = () => {
+ this.log.info("observed that startOver is about to re-initialize the identity");
+ Services.obs.removeObserver(observeStartOverIdentity, "weave:service:start-over:init-identity");
+ // We've now reset all sync prefs - set the engine related prefs back to
+ // what they were.
+ for (let [prefName, prefType, prefVal] of enginePrefs) {
+ this.log.debug("Restoring pref ${prefName} (type=${prefType}) to ${prefVal}",
+ {prefName, prefType, prefVal});
+ switch (prefType) {
+ case Services.prefs.PREF_BOOL:
+ Services.prefs.setBoolPref(prefName, prefVal);
+ break;
+ case Services.prefs.PREF_STRING:
+ Services.prefs.setCharPref(prefName, prefVal);
+ break;
+ default:
+ // _getEngineEnabledPrefs doesn't return any other type...
+ Cu.reportError("unknown engine pref type for " + prefName + ": " + prefType);
+ }
+ }
+ }, "weave:service:start-over:init-identity", false);
+
+ // And another observer for the startOver being fully complete - the only
+ // reason for this is so we can wait until everything is fully reset.
+ let startOverComplete = new Promise((resolve, reject) => {
+ let observe;
+ Services.obs.addObserver(observe = () => {
+ this.log.info("observed that startOver is complete");
+ Services.obs.removeObserver(observe, "weave:service:start-over:finish");
+ resolve();
+ }, "weave:service:start-over:finish", false);
+ });
+
+ Weave.Service.startOver();
+ // need to wait for an observer.
+ Services.obs.notifyObservers(null, OBSERVER_INTERNAL_STATE_CHANGE_TOPIC,
+ this.STATE_INTERNAL_WAITING_START_OVER);
+ yield startOverComplete;
+ // observer fired, now kick things off with the FxA user.
+ this.log.info("scheduling initial FxA sync.");
+ // Note we technically don't need to unblockSync as by now all sync prefs
+ // have been reset - but it doesn't hurt.
+ this._unblockSync();
+ Weave.Service.scheduler.scheduleNextSync(0);
+
+ // Tell the front end that migration is now complete -- Sync is now
+ // configured with an FxA user.
+ forceObserver = true;
+ this.log.info("Migration complete");
+ update(null);
+
+ Services.obs.notifyObservers(null, OBSERVER_INTERNAL_STATE_CHANGE_TOPIC,
+ this.STATE_INTERNAL_COMPLETE);
+ return null;
+ }),
+
+ /* Return an object with the preferences we care about */
+ _getSentinelPrefs() {
+ let result = {};
+ for (let pref of FXA_SENTINEL_PREFS) {
+ if (Services.prefs.prefHasUserValue(pref)) {
+ result[pref] = Services.prefs.getCharPref(pref);
+ }
+ }
+ return result;
+ },
+
+ /* Apply any preferences we've obtained from the sentinel */
+ _applySentinelPrefs(savedPrefs) {
+ for (let pref of FXA_SENTINEL_PREFS) {
+ if (savedPrefs[pref]) {
+ Services.prefs.setCharPref(pref, savedPrefs[pref]);
+ }
+ }
+ },
+
+ /* Ask sync to upload the migration sentinel */
+ _setSyncMigrationSentinel: Task.async(function* () {
+ yield WeaveService.whenLoaded();
+ let signedInUser = yield fxAccounts.getSignedInUser();
+ let sentinel = {
+ email: signedInUser.email,
+ uid: signedInUser.uid,
+ verified: signedInUser.verified,
+ prefs: this._getSentinelPrefs(),
+ };
+ yield Weave.Service.setFxAMigrationSentinel(sentinel);
+ }),
+
+ /* Ask sync to upload the migration sentinal if we (or any other linked device)
+ haven't previously written one.
+ */
+ _setMigrationSentinelIfNecessary: Task.async(function* () {
+ if (!(yield this._getSyncMigrationSentinel())) {
+ this.log.info("writing the migration sentinel");
+ yield this._setSyncMigrationSentinel();
+ }
+ }),
+
+ /* Ask sync to return a migration sentinel if one exists, otherwise return null */
+ _getSyncMigrationSentinel: Task.async(function* () {
+ yield WeaveService.whenLoaded();
+ let sentinel = yield Weave.Service.getFxAMigrationSentinel();
+ this.log.debug("got migration sentinel ${}", sentinel);
+ return sentinel;
+ }),
+
+ _getDefaultAccountName: Task.async(function* (sentinel) {
+ // Requires looking to see if other devices have written a migration
+ // sentinel (eg, see _haveSynchedMigrationSentinel), and if not, see if
+ // the legacy account name appears to be a valid email address (via the
+ // services.sync.account pref), otherwise return null.
+ // NOTE: Sync does all this synchronously via nested event loops, but we
+ // expose a promise to make future migration to an async-sync easier.
+ if (sentinel && sentinel.email) {
+ this.log.info("defaultAccountName found via sentinel: ${}", sentinel.email);
+ return sentinel.email;
+ }
+ // No previous migrations, so check the existing account name.
+ let account = Weave.Service.identity.account;
+ if (account && account.contains("@")) {
+ this.log.info("defaultAccountName found via legacy account name: {}", account);
+ return account;
+ }
+ this.log.info("defaultAccountName could not find an account");
+ return null;
+ }),
+
+ // Prevent sync from automatically starting
+ _blockSync() {
+ Weave.Service.scheduler.blockSync();
+ },
+
+ _unblockSync() {
+ Weave.Service.scheduler.unblockSync();
+ },
+
+ /* Return a list of [prefName, prefType, prefVal] for all engine related
+ preferences.
+ */
+ _getEngineEnabledPrefs() {
+ let result = [];
+ for (let engine of Weave.Service.engineManager.getAll()) {
+ let prefName = "services.sync.engine." + engine.prefName;
+ let prefVal;
+ try {
+ prefVal = Services.prefs.getBoolPref(prefName);
+ result.push([prefName, Services.prefs.PREF_BOOL, prefVal]);
+ } catch (ex) {} /* just skip this pref */
+ }
+ // and the declined list.
+ try {
+ let prefName = "services.sync.declinedEngines";
+ let prefVal = Services.prefs.getCharPref(prefName);
+ result.push([prefName, Services.prefs.PREF_STRING, prefVal]);
+ } catch (ex) {}
+ return result;
+ },
+
+ /* return true if all engines are enabled, false otherwise. */
+ _allEnginesEnabled() {
+ return Weave.Service.engineManager.getAll().every(e => e.enabled);
+ },
+
+ /*
+ * Some helpers for the UI to try and move to the next state.
+ */
+
+ // Open a UI for the user to create a Firefox Account. This should only be
+ // called while we are in the STATE_USER_FXA state. When the user completes
+ // the creation we'll see an ONLOGIN_NOTIFICATION notification from FxA and
+ // we'll move to either the STATE_USER_FXA_VERIFIED state or we'll just
+ // complete the migration if they login as an already verified user.
+ createFxAccount: Task.async(function* (win) {
+ let {url, options} = yield this.getFxAccountCreationOptions();
+ win.switchToTabHavingURI(url, true, options);
+ // An FxA observer will fire when the user completes this, which will
+ // cause us to move to the next "user blocked" state and notify via our
+ // observer notification.
+ }),
+
+ // Returns an object with properties "url" and "options", suitable for
+ // opening FxAccounts to create/signin to FxA suitable for the migration
+ // state. The caller of this is responsible for the actual opening of the
+ // page.
+ // This should only be called while we are in the STATE_USER_FXA state. When
+ // the user completes the creation we'll see an ONLOGIN_NOTIFICATION
+ // notification from FxA and we'll move to either the STATE_USER_FXA_VERIFIED
+ // state or we'll just complete the migration if they login as an already
+ // verified user.
+ getFxAccountCreationOptions: Task.async(function* (win) {
+ // warn if we aren't in the expected state - but go ahead anyway!
+ if (this._state != this.STATE_USER_FXA) {
+ this.log.warn("getFxAccountCreationOptions called in an unexpected state: ${}", this._state);
+ }
+ // We need to obtain the sentinel and apply any prefs that might be
+ // specified *before* attempting to setup FxA as the prefs might
+ // specify custom servers etc.
+ let sentinel = yield this._getSyncMigrationSentinel();
+ if (sentinel && sentinel.prefs) {
+ this._applySentinelPrefs(sentinel.prefs);
+ }
+ // If we already have a sentinel then we assume the user has previously
+ // created the specified account, so just ask to sign-in.
+ let action = sentinel ? "signin" : "signup";
+ // See if we can find a default account name to use.
+ let email = yield this._getDefaultAccountName(sentinel);
+ let tail = email ? "&email=" + encodeURIComponent(email) : "";
+ // A special flag so server-side metrics can tell this is part of migration.
+ tail += "&migration=sync11";
+ // We want to ask FxA to offer a "Customize Sync" checkbox iff any engines
+ // are disabled.
+ let customize = !this._allEnginesEnabled();
+ tail += "&customizeSync=" + customize;
+
+ // We assume the caller of this is going to actually use it, so record
+ // telemetry now.
+ this.recordTelemetry(this.TELEMETRY_ACCEPTED);
+ return {
+ url: "about:accounts?action=" + action + tail,
+ options: {ignoreFragment: true, replaceQueryString: true}
+ };
+ }),
+
+ // Ask the FxA servers to re-send a verification mail for the currently
+ // logged in user. This should only be called while we are in the
+ // STATE_USER_FXA_VERIFIED state. When the user clicks on the link in
+ // the mail we should see an ONVERIFIED_NOTIFICATION which will cause us
+ // to complete the migration.
+ resendVerificationMail: Task.async(function * (win) {
+ // warn if we aren't in the expected state - but go ahead anyway!
+ if (this._state != this.STATE_USER_FXA_VERIFIED) {
+ this.log.warn("resendVerificationMail called in an unexpected state: ${}", this._state);
+ }
+ let ok = true;
+ try {
+ yield fxAccounts.resendVerificationEmail();
+ } catch (ex) {
+ this.log.error("Failed to resend verification mail: ${}", ex);
+ ok = false;
+ }
+ this.recordTelemetry(this.TELEMETRY_ACCEPTED);
+ let fxauser = yield fxAccounts.getSignedInUser();
+ let sb = Services.strings.createBundle("chrome://browser/locale/accounts.properties");
+
+ let heading = ok ?
+ sb.formatStringFromName("verificationSentHeading", [fxauser.email], 1) :
+ sb.GetStringFromName("verificationNotSentHeading");
+ let title = sb.GetStringFromName(ok ? "verificationSentTitle" : "verificationNotSentTitle");
+ let description = sb.GetStringFromName(ok ? "verificationSentDescription"
+ : "verificationNotSentDescription");
+
+ let factory = Cc["@mozilla.org/prompter;1"]
+ .getService(Ci.nsIPromptFactory);
+ let prompt = factory.getPrompt(win, Ci.nsIPrompt);
+ let bag = prompt.QueryInterface(Ci.nsIWritablePropertyBag2);
+ bag.setPropertyAsBool("allowTabModal", true);
+
+ prompt.alert(title, heading + "\n\n" + description);
+ }),
+
+ // "forget" about the current Firefox account. This should only be called
+ // while we are in the STATE_USER_FXA_VERIFIED state. After this we will
+ // see an ONLOGOUT_NOTIFICATION, which will cause the migrator to return back
+ // to the STATE_USER_FXA state, from where they can choose a different account.
+ forgetFxAccount: Task.async(function * () {
+ // warn if we aren't in the expected state - but go ahead anyway!
+ if (this._state != this.STATE_USER_FXA_VERIFIED) {
+ this.log.warn("forgetFxAccount called in an unexpected state: ${}", this._state);
+ }
+ return fxAccounts.signOut();
+ }),
+
+ recordTelemetry(flag) {
+ // Note the value is the telemetry field name - but this is an
+ // implementation detail which could be changed later.
+ switch (flag) {
+ case this.TELEMETRY_ACCEPTED:
+ case this.TELEMETRY_UNLINKED:
+ case this.TELEMETRY_DECLINED:
+ Services.obs.notifyObservers(null, OBSERVER_INTERNAL_TELEMETRY_TOPIC, flag);
+ break;
+ default:
+ throw new Error("Unexpected telemetry flag: " + flag);
+ }
+ },
+
+ get learnMoreLink() {
+ try {
+ var url = Services.prefs.getCharPref("app.support.baseURL");
+ } catch (err) {
+ return null;
+ }
+ url += "sync-upgrade";
+ let sb = Services.strings.createBundle("chrome://weave/locale/services/sync.properties");
+ return {
+ text: sb.GetStringFromName("sync.eol.learnMore.label"),
+ href: Services.urlFormatter.formatURL(url),
+ };
+ },
+};
+
+// We expose a singleton
+this.EXPORTED_SYMBOLS = ["fxaMigrator"];
+let fxaMigrator = new Migrator();
diff --git a/services/sync/modules/SyncedTabs.jsm b/services/sync/modules/SyncedTabs.jsm
deleted file mode 100644
index 1a69e3564..000000000
--- a/services/sync/modules/SyncedTabs.jsm
+++ /dev/null
@@ -1,301 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-"use strict";
-
-this.EXPORTED_SYMBOLS = ["SyncedTabs"];
-
-
-const { classes: Cc, interfaces: Ci, results: Cr, utils: Cu } = Components;
-
-Cu.import("resource://gre/modules/Services.jsm");
-Cu.import("resource://gre/modules/XPCOMUtils.jsm");
-Cu.import("resource://gre/modules/Task.jsm");
-Cu.import("resource://gre/modules/Log.jsm");
-Cu.import("resource://gre/modules/PlacesUtils.jsm", this);
-Cu.import("resource://services-sync/main.js");
-Cu.import("resource://gre/modules/Preferences.jsm");
-
-// The Sync XPCOM service
-XPCOMUtils.defineLazyGetter(this, "weaveXPCService", function() {
- return Cc["@mozilla.org/weave/service;1"]
- .getService(Ci.nsISupports)
- .wrappedJSObject;
-});
-
-// from MDN...
-function escapeRegExp(string) {
- return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
-}
-
-// A topic we fire whenever we have new tabs available. This might be due
-// to a request made by this module to refresh the tab list, or as the result
-// of a regularly scheduled sync. The intent is that consumers just listen
-// for this notification and update their UI in response.
-const TOPIC_TABS_CHANGED = "services.sync.tabs.changed";
-
-// The interval, in seconds, before which we consider the existing list
-// of tabs "fresh enough" and don't force a new sync.
-const TABS_FRESH_ENOUGH_INTERVAL = 30;
-
-let log = Log.repository.getLogger("Sync.RemoteTabs");
-// A new scope to do the logging thang...
-(function() {
- let level = Preferences.get("services.sync.log.logger.tabs");
- if (level) {
- let appender = new Log.DumpAppender();
- log.level = appender.level = Log.Level[level] || Log.Level.Debug;
- log.addAppender(appender);
- }
-})();
-
-
-// A private singleton that does the work.
-let SyncedTabsInternal = {
- /* Make a "tab" record. Returns a promise */
- _makeTab: Task.async(function* (client, tab, url, showRemoteIcons) {
- let icon;
- if (showRemoteIcons) {
- icon = tab.icon;
- }
- if (!icon) {
- try {
- icon = (yield PlacesUtils.promiseFaviconLinkUrl(url)).spec;
- } catch (ex) { /* no favicon avaiable */ }
- }
- if (!icon) {
- icon = "";
- }
- return {
- type: "tab",
- title: tab.title || url,
- url,
- icon,
- client: client.id,
- lastUsed: tab.lastUsed,
- };
- }),
-
- /* Make a "client" record. Returns a promise for consistency with _makeTab */
- _makeClient: Task.async(function* (client) {
- return {
- id: client.id,
- type: "client",
- name: Weave.Service.clientsEngine.getClientName(client.id),
- isMobile: Weave.Service.clientsEngine.isMobile(client.id),
- lastModified: client.lastModified * 1000, // sec to ms
- tabs: []
- };
- }),
-
- _tabMatchesFilter(tab, filter) {
- let reFilter = new RegExp(escapeRegExp(filter), "i");
- return tab.url.match(reFilter) || tab.title.match(reFilter);
- },
-
- getTabClients: Task.async(function* (filter) {
- log.info("Generating tab list with filter", filter);
- let result = [];
-
- // If Sync isn't ready, don't try and get anything.
- if (!weaveXPCService.ready) {
- log.debug("Sync isn't yet ready, so returning an empty tab list");
- return result;
- }
-
- // A boolean that controls whether we should show the icon from the remote tab.
- const showRemoteIcons = Preferences.get("services.sync.syncedTabs.showRemoteIcons", true);
-
- let engine = Weave.Service.engineManager.get("tabs");
-
- let seenURLs = new Set();
- let parentIndex = 0;
- let ntabs = 0;
-
- for (let [guid, client] of Object.entries(engine.getAllClients())) {
- if (!Weave.Service.clientsEngine.remoteClientExists(client.id)) {
- continue;
- }
- let clientRepr = yield this._makeClient(client);
- log.debug("Processing client", clientRepr);
-
- for (let tab of client.tabs) {
- let url = tab.urlHistory[0];
- log.debug("remote tab", url);
- // Note there are some issues with tracking "seen" tabs, including:
- // * We really can't return the entire urlHistory record as we are
- // only checking the first entry - others might be different.
- // * We don't update the |lastUsed| timestamp to reflect the
- // most-recently-seen time.
- // In a followup we should consider simply dropping this |seenUrls|
- // check and return duplicate records - it seems the user will be more
- // confused by tabs not showing up on a device (because it was detected
- // as a dupe so it only appears on a different device) than being
- // confused by seeing the same tab on different clients.
- if (!url || seenURLs.has(url)) {
- continue;
- }
- let tabRepr = yield this._makeTab(client, tab, url, showRemoteIcons);
- if (filter && !this._tabMatchesFilter(tabRepr, filter)) {
- continue;
- }
- seenURLs.add(url);
- clientRepr.tabs.push(tabRepr);
- }
- // We return all clients, even those without tabs - the consumer should
- // filter it if they care.
- ntabs += clientRepr.tabs.length;
- result.push(clientRepr);
- }
- log.info(`Final tab list has ${result.length} clients with ${ntabs} tabs.`);
- return result;
- }),
-
- syncTabs(force) {
- if (!force) {
- // Don't bother refetching tabs if we already did so recently
- let lastFetch = Preferences.get("services.sync.lastTabFetch", 0);
- let now = Math.floor(Date.now() / 1000);
- if (now - lastFetch < TABS_FRESH_ENOUGH_INTERVAL) {
- log.info("_refetchTabs was done recently, do not doing it again");
- return Promise.resolve(false);
- }
- }
-
- // If Sync isn't configured don't try and sync, else we will get reports
- // of a login failure.
- if (Weave.Status.checkSetup() == Weave.CLIENT_NOT_CONFIGURED) {
- log.info("Sync client is not configured, so not attempting a tab sync");
- return Promise.resolve(false);
- }
- // Ask Sync to just do the tabs engine if it can.
- // Sync is currently synchronous, so do it after an event-loop spin to help
- // keep the UI responsive.
- return new Promise((resolve, reject) => {
- Services.tm.currentThread.dispatch(() => {
- try {
- log.info("Doing a tab sync.");
- Weave.Service.sync(["tabs"]);
- resolve(true);
- } catch (ex) {
- log.error("Sync failed", ex);
- reject(ex);
- };
- }, Ci.nsIThread.DISPATCH_NORMAL);
- });
- },
-
- observe(subject, topic, data) {
- log.trace(`observed topic=${topic}, data=${data}, subject=${subject}`);
- switch (topic) {
- case "weave:engine:sync:finish":
- if (data != "tabs") {
- return;
- }
- // The tabs engine just finished syncing
- // Set our lastTabFetch pref here so it tracks both explicit sync calls
- // and normally scheduled ones.
- Preferences.set("services.sync.lastTabFetch", Math.floor(Date.now() / 1000));
- Services.obs.notifyObservers(null, TOPIC_TABS_CHANGED, null);
- break;
- case "weave:service:start-over":
- // start-over needs to notify so consumers find no tabs.
- Preferences.reset("services.sync.lastTabFetch");
- Services.obs.notifyObservers(null, TOPIC_TABS_CHANGED, null);
- break;
- case "nsPref:changed":
- Services.obs.notifyObservers(null, TOPIC_TABS_CHANGED, null);
- break;
- default:
- break;
- }
- },
-
- // Returns true if Sync is configured to Sync tabs, false otherwise
- get isConfiguredToSyncTabs() {
- if (!weaveXPCService.ready) {
- log.debug("Sync isn't yet ready; assuming tab engine is enabled");
- return true;
- }
-
- let engine = Weave.Service.engineManager.get("tabs");
- return engine && engine.enabled;
- },
-
- get hasSyncedThisSession() {
- let engine = Weave.Service.engineManager.get("tabs");
- return engine && engine.hasSyncedThisSession;
- },
-};
-
-Services.obs.addObserver(SyncedTabsInternal, "weave:engine:sync:finish", false);
-Services.obs.addObserver(SyncedTabsInternal, "weave:service:start-over", false);
-// Observe the pref the indicates the state of the tabs engine has changed.
-// This will force consumers to re-evaluate the state of sync and update
-// accordingly.
-Services.prefs.addObserver("services.sync.engine.tabs", SyncedTabsInternal, false);
-
-// The public interface.
-this.SyncedTabs = {
- // A mock-point for tests.
- _internal: SyncedTabsInternal,
-
- // We make the topic for the observer notification public.
- TOPIC_TABS_CHANGED,
-
- // Returns true if Sync is configured to Sync tabs, false otherwise
- get isConfiguredToSyncTabs() {
- return this._internal.isConfiguredToSyncTabs;
- },
-
- // Returns true if a tab sync has completed once this session. If this
- // returns false, then getting back no clients/tabs possibly just means we
- // are waiting for that first sync to complete.
- get hasSyncedThisSession() {
- return this._internal.hasSyncedThisSession;
- },
-
- // Return a promise that resolves with an array of client records, each with
- // a .tabs array. Note that part of the contract for this module is that the
- // returned objects are not shared between invocations, so callers are free
- // to mutate the returned objects (eg, sort, truncate) however they see fit.
- getTabClients(query) {
- return this._internal.getTabClients(query);
- },
-
- // Starts a background request to start syncing tabs. Returns a promise that
- // resolves when the sync is complete, but there's no resolved value -
- // callers should be listening for TOPIC_TABS_CHANGED.
- // If |force| is true we always sync. If false, we only sync if the most
- // recent sync wasn't "recently".
- syncTabs(force) {
- return this._internal.syncTabs(force);
- },
-
- sortTabClientsByLastUsed(clients, maxTabs = Infinity) {
- // First sort and filter the list of tabs for each client. Note that
- // this module promises that the objects it returns are never
- // shared, so we are free to mutate those objects directly.
- for (let client of clients) {
- let tabs = client.tabs;
- tabs.sort((a, b) => b.lastUsed - a.lastUsed);
- if (Number.isFinite(maxTabs)) {
- client.tabs = tabs.slice(0, maxTabs);
- }
- }
- // Now sort the clients - the clients are sorted in the order of the
- // most recent tab for that client (ie, it is important the tabs for
- // each client are already sorted.)
- clients.sort((a, b) => {
- if (a.tabs.length == 0) {
- return 1; // b comes first.
- }
- if (b.tabs.length == 0) {
- return -1; // a comes first.
- }
- return b.tabs[0].lastUsed - a.tabs[0].lastUsed;
- });
- },
-};
-
diff --git a/services/sync/modules/addonsreconciler.js b/services/sync/modules/addonsreconciler.js
index a60fc8d56..2e838e885 100644
--- a/services/sync/modules/addonsreconciler.js
+++ b/services/sync/modules/addonsreconciler.js
@@ -17,7 +17,7 @@
"use strict";
-var Cu = Components.utils;
+const Cu = Components.utils;
Cu.import("resource://gre/modules/Log.jsm");
Cu.import("resource://services-sync/util.js");
@@ -218,12 +218,11 @@ AddonsReconciler.prototype = {
}
this._addons = json.addons;
- for (let id in this._addons) {
- let record = this._addons[id];
+ for each (let record in this._addons) {
record.modified = new Date(record.modified);
}
- for (let [time, change, id] of json.changes) {
+ for each (let [time, change, id] in json.changes) {
this._changes.push([new Date(time), change, id]);
}
@@ -247,9 +246,9 @@ AddonsReconciler.prototype = {
let file = path || DEFAULT_STATE_FILE;
let state = {version: 1, addons: {}, changes: []};
- for (let [id, record] of Object.entries(this._addons)) {
+ for (let [id, record] in Iterator(this._addons)) {
state.addons[id] = {};
- for (let [k, v] of Object.entries(record)) {
+ for (let [k, v] in Iterator(record)) {
if (k == "modified") {
state.addons[id][k] = v.getTime();
}
@@ -259,7 +258,7 @@ AddonsReconciler.prototype = {
}
}
- for (let [time, change, id] of this._changes) {
+ for each (let [time, change, id] in this._changes) {
state.changes.push([time.getTime(), change, id]);
}
@@ -351,14 +350,14 @@ AddonsReconciler.prototype = {
AddonManager.getAllAddons(function (addons) {
let ids = {};
- for (let addon of addons) {
+ for each (let addon in addons) {
ids[addon.id] = true;
this.rectifyStateFromAddon(addon);
}
// Look for locally-defined add-ons that no longer exist and update their
// record.
- for (let [id, addon] of Object.entries(this._addons)) {
+ for (let [id, addon] in Iterator(this._addons)) {
if (id in ids) {
continue;
}
@@ -374,7 +373,7 @@ AddonsReconciler.prototype = {
}
let installFound = false;
- for (let install of installs) {
+ for each (let install in installs) {
if (install.addon && install.addon.id == id &&
install.state == AddonManager.STATE_INSTALLED) {
@@ -417,7 +416,7 @@ AddonsReconciler.prototype = {
* Addon instance being updated.
*/
rectifyStateFromAddon: function rectifyStateFromAddon(addon) {
- this._log.debug(`Rectifying state for addon ${addon.name} (version=${addon.version}, id=${addon.id})`);
+ this._log.debug("Rectifying state for addon: " + addon.id);
this._ensureStateLoaded();
let id = addon.id;
@@ -434,8 +433,7 @@ AddonsReconciler.prototype = {
modified: now,
type: addon.type,
scope: addon.scope,
- foreignInstall: addon.foreignInstall,
- isSyncable: addon.isSyncable,
+ foreignInstall: addon.foreignInstall
};
this._addons[id] = record;
this._log.debug("Adding change because add-on not present locally: " +
@@ -445,7 +443,6 @@ AddonsReconciler.prototype = {
}
let record = this._addons[id];
- record.isSyncable = addon.isSyncable;
if (!record.installed) {
// It is possible the record is marked as uninstalled because an
@@ -486,11 +483,12 @@ AddonsReconciler.prototype = {
this._log.info("Change recorded for " + state.id);
this._changes.push([date, change, state.id]);
- for (let listener of this._listeners) {
+ for each (let listener in this._listeners) {
try {
listener.changeListener.call(listener, date, change, state);
} catch (ex) {
- this._log.warn("Exception calling change listener", ex);
+ this._log.warn("Exception calling change listener: " +
+ Utils.exceptionStr(ex));
}
}
},
@@ -556,8 +554,7 @@ AddonsReconciler.prototype = {
* @return Object on success on null on failure.
*/
getAddonStateFromSyncGUID: function getAddonStateFromSyncGUID(guid) {
- for (let id in this.addons) {
- let addon = this.addons[id];
+ for each (let addon in this.addons) {
if (addon.guid == guid) {
return addon;
}
@@ -636,7 +633,7 @@ AddonsReconciler.prototype = {
}
}
catch (ex) {
- this._log.warn("Exception", ex);
+ this._log.warn("Exception: " + Utils.exceptionStr(ex));
}
},
diff --git a/services/sync/modules/addonutils.js b/services/sync/modules/addonutils.js
index 95da6be0a..54b441b9e 100644
--- a/services/sync/modules/addonutils.js
+++ b/services/sync/modules/addonutils.js
@@ -6,7 +6,7 @@
this.EXPORTED_SYMBOLS = ["AddonUtils"];
-var {interfaces: Ci, utils: Cu} = Components;
+const {interfaces: Ci, utils: Cu} = Components;
Cu.import("resource://gre/modules/XPCOMUtils.jsm");
Cu.import("resource://gre/modules/Log.jsm");
@@ -38,10 +38,21 @@ AddonUtilsInternal.prototype = {
* Function to be called with result of operation.
*/
getInstallFromSearchResult:
- function getInstallFromSearchResult(addon, cb) {
+ function getInstallFromSearchResult(addon, cb, requireSecureURI=true) {
this._log.debug("Obtaining install for " + addon.id);
+ // Verify that the source URI uses TLS. We don't allow installs from
+ // insecure sources for security reasons. The Addon Manager ensures that
+ // cert validation, etc is performed.
+ if (requireSecureURI) {
+ let scheme = addon.sourceURI.scheme;
+ if (scheme != "https") {
+ cb(new Error("Insecure source URI scheme: " + scheme), addon.install);
+ return;
+ }
+ }
+
// We should theoretically be able to obtain (and use) addon.install if
// it is available. However, the addon.sourceURI rewriting won't be
// reflected in the AddonInstall, so we can't use it. If we ever get rid
@@ -69,6 +80,8 @@ AddonUtilsInternal.prototype = {
* syncGUID - Sync GUID to use for the new add-on.
* enabled - Boolean indicating whether the add-on should be enabled upon
* install.
+ * requireSecureURI - Boolean indicating whether to require a secure
+ * URI to install from. This defaults to true.
*
* When complete it calls a callback with 2 arguments, error and result.
*
@@ -92,6 +105,10 @@ AddonUtilsInternal.prototype = {
function installAddonFromSearchResult(addon, options, cb) {
this._log.info("Trying to install add-on from search result: " + addon.id);
+ if (options.requireSecureURI === undefined) {
+ options.requireSecureURI = true;
+ }
+
this.getInstallFromSearchResult(addon, function onResult(error, install) {
if (error) {
cb(error, null);
@@ -147,10 +164,10 @@ AddonUtilsInternal.prototype = {
install.install();
}
catch (ex) {
- this._log.error("Error installing add-on", ex);
+ this._log.error("Error installing add-on: " + Utils.exceptionstr(ex));
cb(ex, null);
}
- }.bind(this));
+ }.bind(this), options.requireSecureURI);
},
/**
@@ -231,7 +248,7 @@ AddonUtilsInternal.prototype = {
}
let ids = [];
- for (let addon of installs) {
+ for each (let addon in installs) {
ids.push(addon.id);
}
@@ -244,7 +261,6 @@ AddonUtilsInternal.prototype = {
installedIDs: [],
installs: [],
addons: [],
- skipped: [],
errors: []
};
@@ -282,21 +298,15 @@ AddonUtilsInternal.prototype = {
// server-side metrics aren't skewed (bug 708134). The server should
// ideally send proper URLs, but this solution was deemed too
// complicated at the time the functionality was implemented.
- for (let addon of addons) {
- // Find the specified options for this addon.
- let options;
- for (let install of installs) {
- if (install.id == addon.id) {
- options = install;
- break;
- }
- }
- if (!this.canInstallAddon(addon, options)) {
- ourResult.skipped.push(addon.id);
+ for each (let addon in addons) {
+ // sourceURI presence isn't enforced by AddonRepository. So, we skip
+ // add-ons without a sourceURI.
+ if (!addon.sourceURI) {
+ this._log.info("Skipping install of add-on because missing " +
+ "sourceURI: " + addon.id);
continue;
}
- // We can go ahead and attempt to install it.
toInstall.push(addon);
// We should always be able to QI the nsIURI to nsIURL. If not, we
@@ -332,9 +342,9 @@ AddonUtilsInternal.prototype = {
// Start all the installs asynchronously. They will report back to us
// as they finish, eventually triggering the global callback.
- for (let addon of toInstall) {
+ for each (let addon in toInstall) {
let options = {};
- for (let install of installs) {
+ for each (let install in installs) {
if (install.id == addon.id) {
options = install;
break;
@@ -353,51 +363,9 @@ AddonUtilsInternal.prototype = {
},
/**
- * Returns true if we are able to install the specified addon, false
- * otherwise. It is expected that this will log the reason if it returns
- * false.
- *
- * @param addon
- * (Addon) Add-on instance to check.
- * @param options
- * (object) The options specified for this addon. See installAddons()
- * for the valid elements.
- */
- canInstallAddon(addon, options) {
- // sourceURI presence isn't enforced by AddonRepository. So, we skip
- // add-ons without a sourceURI.
- if (!addon.sourceURI) {
- this._log.info("Skipping install of add-on because missing " +
- "sourceURI: " + addon.id);
- return false;
- }
- // Verify that the source URI uses TLS. We don't allow installs from
- // insecure sources for security reasons. The Addon Manager ensures
- // that cert validation etc is performed.
- // (We should also consider just dropping this entirely and calling
- // XPIProvider.isInstallAllowed, but that has additional semantics we might
- // need to think through...)
- let requireSecureURI = true;
- if (options && options.requireSecureURI !== undefined) {
- requireSecureURI = options.requireSecureURI;
- }
-
- if (requireSecureURI) {
- let scheme = addon.sourceURI.scheme;
- if (scheme != "https") {
- this._log.info(`Skipping install of add-on "${addon.id}" because sourceURI's scheme of "${scheme}" is not trusted`);
- return false;
- }
- }
- this._log.info(`Add-on "${addon.id}" is able to be installed`);
- return true;
- },
-
-
- /**
* Update the user disabled flag for an add-on.
*
- * The supplied callback will be called when the operation is
+ * The supplied callback will ba called when the operation is
* complete. If the new flag matches the existing or if the add-on
* isn't currently active, the function will fire the callback
* immediately. Else, the callback is invoked when the AddonManager
diff --git a/services/sync/modules/bookmark_validator.js b/services/sync/modules/bookmark_validator.js
deleted file mode 100644
index 2a94ba043..000000000
--- a/services/sync/modules/bookmark_validator.js
+++ /dev/null
@@ -1,784 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this file,
- * You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-"use strict";
-
-const Cu = Components.utils;
-
-Cu.import("resource://gre/modules/PlacesUtils.jsm");
-Cu.import("resource://gre/modules/PlacesSyncUtils.jsm");
-Cu.import("resource://gre/modules/Task.jsm");
-Cu.import("resource://gre/modules/XPCOMUtils.jsm");
-
-
-this.EXPORTED_SYMBOLS = ["BookmarkValidator", "BookmarkProblemData"];
-
-const LEFT_PANE_ROOT_ANNO = "PlacesOrganizer/OrganizerFolder";
-const LEFT_PANE_QUERY_ANNO = "PlacesOrganizer/OrganizerQuery";
-
-// Indicates if a local bookmark tree node should be excluded from syncing.
-function isNodeIgnored(treeNode) {
- return treeNode.annos && treeNode.annos.some(anno => anno.name == LEFT_PANE_ROOT_ANNO ||
- anno.name == LEFT_PANE_QUERY_ANNO);
-}
-const BOOKMARK_VALIDATOR_VERSION = 1;
-
-/**
- * Result of bookmark validation. Contains the following fields which describe
- * server-side problems unless otherwise specified.
- *
- * - missingIDs (number): # of objects with missing ids
- * - duplicates (array of ids): ids seen more than once
- * - parentChildMismatches (array of {parent: parentid, child: childid}):
- * instances where the child's parentid and the parent's children array
- * do not match
- * - cycles (array of array of ids). List of cycles found in the server-side tree.
- * - clientCycles (array of array of ids). List of cycles found in the client-side tree.
- * - orphans (array of {id: string, parent: string}): List of nodes with
- * either no parentid, or where the parent could not be found.
- * - missingChildren (array of {parent: id, child: id}):
- * List of parent/children where the child id couldn't be found
- * - deletedChildren (array of { parent: id, child: id }):
- * List of parent/children where child id was a deleted item (but still showed up
- * in the children array)
- * - multipleParents (array of {child: id, parents: array of ids}):
- * List of children that were part of multiple parent arrays
- * - deletedParents (array of ids) : List of records that aren't deleted but
- * had deleted parents
- * - childrenOnNonFolder (array of ids): list of non-folders that still have
- * children arrays
- * - duplicateChildren (array of ids): list of records who have the same
- * child listed multiple times in their children array
- * - parentNotFolder (array of ids): list of records that have parents that
- * aren't folders
- * - rootOnServer (boolean): true if the root came from the server
- * - badClientRoots (array of ids): Contains any client-side root ids where
- * the root is missing or isn't a (direct) child of the places root.
- *
- * - clientMissing: Array of ids on the server missing from the client
- * - serverMissing: Array of ids on the client missing from the server
- * - serverDeleted: Array of ids on the client that the server had marked as deleted.
- * - serverUnexpected: Array of ids that appear on the server but shouldn't
- * because the client attempts to never upload them.
- * - differences: Array of {id: string, differences: string array} recording
- * the non-structural properties that are differente between the client and server
- * - structuralDifferences: As above, but contains the items where the differences were
- * structural, that is, they contained childGUIDs or parentid
- */
-class BookmarkProblemData {
- constructor() {
- this.rootOnServer = false;
- this.missingIDs = 0;
-
- this.duplicates = [];
- this.parentChildMismatches = [];
- this.cycles = [];
- this.clientCycles = [];
- this.orphans = [];
- this.missingChildren = [];
- this.deletedChildren = [];
- this.multipleParents = [];
- this.deletedParents = [];
- this.childrenOnNonFolder = [];
- this.duplicateChildren = [];
- this.parentNotFolder = [];
-
- this.badClientRoots = [];
- this.clientMissing = [];
- this.serverMissing = [];
- this.serverDeleted = [];
- this.serverUnexpected = [];
- this.differences = [];
- this.structuralDifferences = [];
- }
-
- /**
- * Convert ("difference", [{ differences: ["tags", "name"] }, { differences: ["name"] }]) into
- * [{ name: "difference:tags", count: 1}, { name: "difference:name", count: 2 }], etc.
- */
- _summarizeDifferences(prefix, diffs) {
- let diffCounts = new Map();
- for (let { differences } of diffs) {
- for (let type of differences) {
- let name = prefix + ":" + type;
- let count = diffCounts.get(name) || 0;
- diffCounts.set(name, count + 1);
- }
- }
- return [...diffCounts].map(([name, count]) => ({ name, count }));
- }
-
- /**
- * Produce a list summarizing problems found. Each entry contains {name, count},
- * where name is the field name for the problem, and count is the number of times
- * the problem was encountered.
- *
- * Validation has failed if all counts are not 0.
- *
- * If the `full` argument is truthy, we also include information about which
- * properties we saw structural differences in. Currently, this means either
- * "sdiff:parentid" and "sdiff:childGUIDS" may be present.
- */
- getSummary(full) {
- let result = [
- { name: "clientMissing", count: this.clientMissing.length },
- { name: "serverMissing", count: this.serverMissing.length },
- { name: "serverDeleted", count: this.serverDeleted.length },
- { name: "serverUnexpected", count: this.serverUnexpected.length },
-
- { name: "structuralDifferences", count: this.structuralDifferences.length },
- { name: "differences", count: this.differences.length },
-
- { name: "missingIDs", count: this.missingIDs },
- { name: "rootOnServer", count: this.rootOnServer ? 1 : 0 },
-
- { name: "duplicates", count: this.duplicates.length },
- { name: "parentChildMismatches", count: this.parentChildMismatches.length },
- { name: "cycles", count: this.cycles.length },
- { name: "clientCycles", count: this.clientCycles.length },
- { name: "badClientRoots", count: this.badClientRoots.length },
- { name: "orphans", count: this.orphans.length },
- { name: "missingChildren", count: this.missingChildren.length },
- { name: "deletedChildren", count: this.deletedChildren.length },
- { name: "multipleParents", count: this.multipleParents.length },
- { name: "deletedParents", count: this.deletedParents.length },
- { name: "childrenOnNonFolder", count: this.childrenOnNonFolder.length },
- { name: "duplicateChildren", count: this.duplicateChildren.length },
- { name: "parentNotFolder", count: this.parentNotFolder.length },
- ];
- if (full) {
- let structural = this._summarizeDifferences("sdiff", this.structuralDifferences);
- result.push.apply(result, structural);
- }
- return result;
- }
-}
-
-// Defined lazily to avoid initializing PlacesUtils.bookmarks too soon.
-XPCOMUtils.defineLazyGetter(this, "SYNCED_ROOTS", () => [
- PlacesUtils.bookmarks.menuGuid,
- PlacesUtils.bookmarks.toolbarGuid,
- PlacesUtils.bookmarks.unfiledGuid,
- PlacesUtils.bookmarks.mobileGuid,
-]);
-
-class BookmarkValidator {
-
- _followQueries(recordMap) {
- for (let [guid, entry] of recordMap) {
- if (entry.type !== "query" && (!entry.bmkUri || !entry.bmkUri.startsWith("place:"))) {
- continue;
- }
- // Might be worth trying to parse the place: query instead so that this
- // works "automatically" with things like aboutsync.
- let queryNodeParent = PlacesUtils.getFolderContents(entry, false, true);
- if (!queryNodeParent || !queryNodeParent.root.hasChildren) {
- continue;
- }
- queryNodeParent = queryNodeParent.root;
- let queryNode = null;
- let numSiblings = 0;
- let containerWasOpen = queryNodeParent.containerOpen;
- queryNodeParent.containerOpen = true;
- try {
- try {
- numSiblings = queryNodeParent.childCount;
- } catch (e) {
- // This throws when we can't actually get the children. This is the
- // case for history containers, tag queries, ...
- continue;
- }
- for (let i = 0; i < numSiblings && !queryNode; ++i) {
- let child = queryNodeParent.getChild(i);
- if (child && child.bookmarkGuid && child.bookmarkGuid === guid) {
- queryNode = child;
- }
- }
- } finally {
- queryNodeParent.containerOpen = containerWasOpen;
- }
- if (!queryNode) {
- continue;
- }
-
- let concreteId = PlacesUtils.getConcreteItemGuid(queryNode);
- if (!concreteId) {
- continue;
- }
- let concreteItem = recordMap.get(concreteId);
- if (!concreteItem) {
- continue;
- }
- entry.concrete = concreteItem;
- }
- }
-
- createClientRecordsFromTree(clientTree) {
- // Iterate over the treeNode, converting it to something more similar to what
- // the server stores.
- let records = [];
- let recordsByGuid = new Map();
- let syncedRoots = SYNCED_ROOTS;
- function traverse(treeNode, synced) {
- if (!synced) {
- synced = syncedRoots.includes(treeNode.guid);
- } else if (isNodeIgnored(treeNode)) {
- synced = false;
- }
- let guid = PlacesSyncUtils.bookmarks.guidToSyncId(treeNode.guid);
- let itemType = 'item';
- treeNode.ignored = !synced;
- treeNode.id = guid;
- switch (treeNode.type) {
- case PlacesUtils.TYPE_X_MOZ_PLACE:
- let query = null;
- if (treeNode.annos && treeNode.uri.startsWith("place:")) {
- query = treeNode.annos.find(({name}) =>
- name === PlacesSyncUtils.bookmarks.SMART_BOOKMARKS_ANNO);
- }
- if (query && query.value) {
- itemType = 'query';
- } else {
- itemType = 'bookmark';
- }
- break;
- case PlacesUtils.TYPE_X_MOZ_PLACE_CONTAINER:
- let isLivemark = false;
- if (treeNode.annos) {
- for (let anno of treeNode.annos) {
- if (anno.name === PlacesUtils.LMANNO_FEEDURI) {
- isLivemark = true;
- treeNode.feedUri = anno.value;
- } else if (anno.name === PlacesUtils.LMANNO_SITEURI) {
- isLivemark = true;
- treeNode.siteUri = anno.value;
- }
- }
- }
- itemType = isLivemark ? "livemark" : "folder";
- break;
- case PlacesUtils.TYPE_X_MOZ_PLACE_SEPARATOR:
- itemType = 'separator';
- break;
- }
-
- if (treeNode.tags) {
- treeNode.tags = treeNode.tags.split(",");
- } else {
- treeNode.tags = [];
- }
- treeNode.type = itemType;
- treeNode.pos = treeNode.index;
- treeNode.bmkUri = treeNode.uri;
- records.push(treeNode);
- // We want to use the "real" guid here.
- recordsByGuid.set(treeNode.guid, treeNode);
- if (treeNode.type === 'folder') {
- treeNode.childGUIDs = [];
- if (!treeNode.children) {
- treeNode.children = [];
- }
- for (let child of treeNode.children) {
- traverse(child, synced);
- child.parent = treeNode;
- child.parentid = guid;
- treeNode.childGUIDs.push(child.guid);
- }
- }
- }
- traverse(clientTree, false);
- clientTree.id = 'places';
- this._followQueries(recordsByGuid);
- return records;
- }
-
- /**
- * Process the server-side list. Mainly this builds the records into a tree,
- * but it also records information about problems, and produces arrays of the
- * deleted and non-deleted nodes.
- *
- * Returns an object containing:
- * - records:Array of non-deleted records. Each record contains the following
- * properties
- * - childGUIDs (array of strings, only present if type is 'folder'): the
- * list of child GUIDs stored on the server.
- * - children (array of records, only present if type is 'folder'):
- * each record has these same properties. This may differ in content
- * from what you may expect from the childGUIDs list, as it won't
- * contain any records that could not be found.
- * - parent (record): The parent to this record.
- * - Unchanged properties send down from the server: id, title, type,
- * parentName, parentid, bmkURI, keyword, tags, pos, queryId, loadInSidebar
- * - root: Root of the server-side bookmark tree. Has the same properties as
- * above.
- * - deletedRecords: As above, but only contains items that the server sent
- * where it also sent indication that the item should be deleted.
- * - problemData: a BookmarkProblemData object, with the caveat that
- * the fields describing client/server relationship will not have been filled
- * out yet.
- */
- inspectServerRecords(serverRecords) {
- let deletedItemIds = new Set();
- let idToRecord = new Map();
- let deletedRecords = [];
-
- let folders = [];
- let problems = [];
-
- let problemData = new BookmarkProblemData();
-
- let resultRecords = [];
-
- for (let record of serverRecords) {
- if (!record.id) {
- ++problemData.missingIDs;
- continue;
- }
- if (record.deleted) {
- deletedItemIds.add(record.id);
- } else {
- if (idToRecord.has(record.id)) {
- problemData.duplicates.push(record.id);
- continue;
- }
- }
- idToRecord.set(record.id, record);
-
- if (record.children) {
- if (record.type !== "folder") {
- // Due to implementation details in engines/bookmarks.js, (Livemark
- // subclassing BookmarkFolder) Livemarks will have a children array,
- // but it should still be empty.
- if (!record.children.length) {
- continue;
- }
- // Otherwise we mark it as an error and still try to resolve the children
- problemData.childrenOnNonFolder.push(record.id);
- }
- folders.push(record);
-
- if (new Set(record.children).size !== record.children.length) {
- problemData.duplicateChildren.push(record.id)
- }
-
- // The children array stores special guids as their local guid values,
- // e.g. 'menu________' instead of 'menu', but all other parts of the
- // serverside bookmark info stores it as the special value ('menu').
- record.childGUIDs = record.children;
- record.children = record.children.map(childID => {
- return PlacesSyncUtils.bookmarks.guidToSyncId(childID);
- });
- }
- }
-
- for (let deletedId of deletedItemIds) {
- let record = idToRecord.get(deletedId);
- if (record && !record.isDeleted) {
- deletedRecords.push(record);
- record.isDeleted = true;
- }
- }
-
- let root = idToRecord.get('places');
-
- if (!root) {
- // Fabricate a root. We want to remember that it's fake so that we can
- // avoid complaining about stuff like it missing it's childGUIDs later.
- root = { id: 'places', children: [], type: 'folder', title: '', fake: true };
- resultRecords.push(root);
- idToRecord.set('places', root);
- } else {
- problemData.rootOnServer = true;
- }
-
- // Build the tree, find orphans, and record most problems having to do with
- // the tree structure.
- for (let [id, record] of idToRecord) {
- if (record === root) {
- continue;
- }
-
- if (record.isDeleted) {
- continue;
- }
-
- let parentID = record.parentid;
- if (!parentID) {
- problemData.orphans.push({id: record.id, parent: parentID});
- continue;
- }
-
- let parent = idToRecord.get(parentID);
- if (!parent) {
- problemData.orphans.push({id: record.id, parent: parentID});
- continue;
- }
-
- if (parent.type !== 'folder') {
- problemData.parentNotFolder.push(record.id);
- if (!parent.children) {
- parent.children = [];
- }
- if (!parent.childGUIDs) {
- parent.childGUIDs = [];
- }
- }
-
- if (!record.isDeleted) {
- resultRecords.push(record);
- }
-
- record.parent = parent;
- if (parent !== root || problemData.rootOnServer) {
- let childIndex = parent.children.indexOf(id);
- if (childIndex < 0) {
- problemData.parentChildMismatches.push({parent: parent.id, child: record.id});
- } else {
- parent.children[childIndex] = record;
- }
- } else {
- parent.children.push(record);
- }
-
- if (parent.isDeleted && !record.isDeleted) {
- problemData.deletedParents.push(record.id);
- }
-
- // We used to check if the parentName on the server matches the actual
- // local parent name, but given this is used only for de-duping a record
- // the first time it is seen and expensive to keep up-to-date, we decided
- // to just stop recording it. See bug 1276969 for more.
- }
-
- // Check that we aren't missing any children.
- for (let folder of folders) {
- folder.unfilteredChildren = folder.children;
- folder.children = [];
- for (let ci = 0; ci < folder.unfilteredChildren.length; ++ci) {
- let child = folder.unfilteredChildren[ci];
- let childObject;
- if (typeof child == "string") {
- // This can happen the parent refers to a child that has a different
- // parentid, or if it refers to a missing or deleted child. It shouldn't
- // be possible with totally valid bookmarks.
- childObject = idToRecord.get(child);
- if (!childObject) {
- problemData.missingChildren.push({parent: folder.id, child});
- } else {
- folder.unfilteredChildren[ci] = childObject;
- if (childObject.isDeleted) {
- problemData.deletedChildren.push({ parent: folder.id, child });
- }
- }
- } else {
- childObject = child;
- }
-
- if (!childObject) {
- continue;
- }
-
- if (childObject.parentid === folder.id) {
- folder.children.push(childObject);
- continue;
- }
-
- // The child is very probably in multiple `children` arrays --
- // see if we already have a problem record about it.
- let currentProblemRecord = problemData.multipleParents.find(pr =>
- pr.child === child);
-
- if (currentProblemRecord) {
- currentProblemRecord.parents.push(folder.id);
- continue;
- }
-
- let otherParent = idToRecord.get(childObject.parentid);
- // it's really an ... orphan ... sort of.
- if (!otherParent) {
- // if we never end up adding to this parent's list, we filter it out after this loop.
- problemData.multipleParents.push({
- child,
- parents: [folder.id]
- });
- if (!problemData.orphans.some(r => r.id === child)) {
- problemData.orphans.push({
- id: child,
- parent: childObject.parentid
- });
- }
- continue;
- }
-
- if (otherParent.isDeleted) {
- if (!problemData.deletedParents.includes(child)) {
- problemData.deletedParents.push(child);
- }
- continue;
- }
-
- if (otherParent.childGUIDs && !otherParent.childGUIDs.includes(child)) {
- if (!problemData.parentChildMismatches.some(r => r.child === child)) {
- // Might not be possible to get here.
- problemData.parentChildMismatches.push({ child, parent: folder.id });
- }
- }
-
- problemData.multipleParents.push({
- child,
- parents: [childObject.parentid, folder.id]
- });
- }
- }
- problemData.multipleParents = problemData.multipleParents.filter(record =>
- record.parents.length >= 2);
-
- problemData.cycles = this._detectCycles(resultRecords);
-
- return {
- deletedRecords,
- records: resultRecords,
- problemData,
- root,
- };
- }
-
- // helper for inspectServerRecords
- _detectCycles(records) {
- // currentPath and pathLookup contain the same data. pathLookup is faster to
- // query, but currentPath gives is the order of traversal that we need in
- // order to report the members of the cycles.
- let pathLookup = new Set();
- let currentPath = [];
- let cycles = [];
- let seenEver = new Set();
- const traverse = node => {
- if (pathLookup.has(node)) {
- let cycleStart = currentPath.lastIndexOf(node);
- let cyclePath = currentPath.slice(cycleStart).map(n => n.id);
- cycles.push(cyclePath);
- return;
- } else if (seenEver.has(node)) {
- // If we're checking the server, this is a problem, but it should already be reported.
- // On the client, this could happen due to including `node.concrete` in the child list.
- return;
- }
- seenEver.add(node);
- let children = node.children || [];
- if (node.concrete) {
- children.push(node.concrete);
- }
- if (children) {
- pathLookup.add(node);
- currentPath.push(node);
- for (let child of children) {
- traverse(child);
- }
- currentPath.pop();
- pathLookup.delete(node);
- }
- };
- for (let record of records) {
- if (!seenEver.has(record)) {
- traverse(record);
- }
- }
-
- return cycles;
- }
-
- // Perform client-side sanity checking that doesn't involve server data
- _validateClient(problemData, clientRecords) {
- problemData.clientCycles = this._detectCycles(clientRecords);
- for (let rootGUID of SYNCED_ROOTS) {
- let record = clientRecords.find(record =>
- record.guid === rootGUID);
- if (!record || record.parentid !== "places") {
- problemData.badClientRoots.push(rootGUID);
- }
- }
- }
-
- /**
- * Compare the list of server records with the client tree.
- *
- * Returns the same data as described in the inspectServerRecords comment,
- * with the following additional fields.
- * - clientRecords: an array of client records in a similar format to
- * the .records (ie, server records) entry.
- * - problemData is the same as for inspectServerRecords, except all properties
- * will be filled out.
- */
- compareServerWithClient(serverRecords, clientTree) {
-
- let clientRecords = this.createClientRecordsFromTree(clientTree);
- let inspectionInfo = this.inspectServerRecords(serverRecords);
- inspectionInfo.clientRecords = clientRecords;
-
- // Mainly do this to remove deleted items and normalize child guids.
- serverRecords = inspectionInfo.records;
- let problemData = inspectionInfo.problemData;
-
- this._validateClient(problemData, clientRecords);
-
- let matches = [];
-
- let allRecords = new Map();
- let serverDeletedLookup = new Set(inspectionInfo.deletedRecords.map(r => r.id));
-
- for (let sr of serverRecords) {
- if (sr.fake) {
- continue;
- }
- allRecords.set(sr.id, {client: null, server: sr});
- }
-
- for (let cr of clientRecords) {
- let unified = allRecords.get(cr.id);
- if (!unified) {
- allRecords.set(cr.id, {client: cr, server: null});
- } else {
- unified.client = cr;
- }
- }
-
-
- for (let [id, {client, server}] of allRecords) {
- if (!client && server) {
- problemData.clientMissing.push(id);
- continue;
- }
- if (!server && client) {
- if (serverDeletedLookup.has(id)) {
- problemData.serverDeleted.push(id);
- } else if (!client.ignored && client.id != "places") {
- problemData.serverMissing.push(id);
- }
- continue;
- }
- if (server && client && client.ignored) {
- problemData.serverUnexpected.push(id);
- }
- let differences = [];
- let structuralDifferences = [];
-
- // Don't bother comparing titles of roots. It's okay if locally it's
- // "Mobile Bookmarks", but the server thinks it's "mobile".
- // TODO: We probably should be handing other localized bookmarks (e.g.
- // default bookmarks) here as well, see bug 1316041.
- if (!SYNCED_ROOTS.includes(client.guid)) {
- // We want to treat undefined, null and an empty string as identical
- if ((client.title || "") !== (server.title || "")) {
- differences.push("title");
- }
- }
-
- if (client.parentid || server.parentid) {
- if (client.parentid !== server.parentid) {
- structuralDifferences.push('parentid');
- }
- }
-
- if (client.tags || server.tags) {
- let cl = client.tags || [];
- let sl = server.tags || [];
- if (cl.length !== sl.length || !cl.every((tag, i) => sl.indexOf(tag) >= 0)) {
- differences.push('tags');
- }
- }
-
- let sameType = client.type === server.type;
- if (!sameType) {
- if (server.type === "query" && client.type === "bookmark" && client.bmkUri.startsWith("place:")) {
- sameType = true;
- }
- }
-
-
- if (!sameType) {
- differences.push('type');
- } else {
- switch (server.type) {
- case 'bookmark':
- case 'query':
- if (server.bmkUri !== client.bmkUri) {
- differences.push('bmkUri');
- }
- break;
- case "livemark":
- if (server.feedUri != client.feedUri) {
- differences.push("feedUri");
- }
- if (server.siteUri != client.siteUri) {
- differences.push("siteUri");
- }
- break;
- case 'folder':
- if (server.id === 'places' && !problemData.rootOnServer) {
- // It's the fabricated places root. It won't have the GUIDs, but
- // it doesn't matter.
- break;
- }
- if (client.childGUIDs || server.childGUIDs) {
- let cl = client.childGUIDs || [];
- let sl = server.childGUIDs || [];
- if (cl.length !== sl.length || !cl.every((id, i) => sl[i] === id)) {
- structuralDifferences.push('childGUIDs');
- }
- }
- break;
- }
- }
-
- if (differences.length) {
- problemData.differences.push({id, differences});
- }
- if (structuralDifferences.length) {
- problemData.structuralDifferences.push({ id, differences: structuralDifferences });
- }
- }
- return inspectionInfo;
- }
-
- _getServerState(engine) {
- let collection = engine.itemSource();
- let collectionKey = engine.service.collectionKeys.keyForCollection(engine.name);
- collection.full = true;
- let items = [];
- collection.recordHandler = function(item) {
- item.decrypt(collectionKey);
- items.push(item.cleartext);
- };
- let resp = collection.getBatched();
- if (!resp.success) {
- throw resp;
- }
- return items;
- }
-
- validate(engine) {
- let self = this;
- return Task.spawn(function*() {
- let start = Date.now();
- let clientTree = yield PlacesUtils.promiseBookmarksTree("", {
- includeItemIds: true
- });
- let serverState = self._getServerState(engine);
- let serverRecordCount = serverState.length;
- let result = self.compareServerWithClient(serverState, clientTree);
- let end = Date.now();
- let duration = end-start;
- return {
- duration,
- version: self.version,
- problems: result.problemData,
- recordCount: serverRecordCount
- };
- });
- }
-
-};
-
-BookmarkValidator.prototype.version = BOOKMARK_VALIDATOR_VERSION;
-
diff --git a/services/sync/modules/browserid_identity.js b/services/sync/modules/browserid_identity.js
index db3821518..bc8ea6b30 100644
--- a/services/sync/modules/browserid_identity.js
+++ b/services/sync/modules/browserid_identity.js
@@ -4,9 +4,9 @@
"use strict";
-this.EXPORTED_SYMBOLS = ["BrowserIDManager", "AuthenticationError"];
+this.EXPORTED_SYMBOLS = ["BrowserIDManager"];
-var {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
+const {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
Cu.import("resource://gre/modules/Log.jsm");
Cu.import("resource://services-common/async.js");
@@ -39,13 +39,12 @@ XPCOMUtils.defineLazyGetter(this, 'log', function() {
});
// FxAccountsCommon.js doesn't use a "namespace", so create one here.
-var fxAccountsCommon = {};
+let fxAccountsCommon = {};
Cu.import("resource://gre/modules/FxAccountsCommon.js", fxAccountsCommon);
const OBSERVER_TOPICS = [
fxAccountsCommon.ONLOGIN_NOTIFICATION,
fxAccountsCommon.ONLOGOUT_NOTIFICATION,
- fxAccountsCommon.ON_ACCOUNT_STATE_CHANGE_NOTIFICATION,
];
const PREF_SYNC_SHOW_CUSTOMIZATION = "services.sync-setup.ui.showCustomizationDialog";
@@ -66,9 +65,8 @@ function deriveKeyBundle(kB) {
some other error object (which should do the right thing when toString() is
called on it)
*/
-function AuthenticationError(details, source) {
+function AuthenticationError(details) {
this.details = details;
- this.source = source;
}
AuthenticationError.prototype = {
@@ -106,6 +104,12 @@ this.BrowserIDManager.prototype = {
// we don't consider the lack of a keybundle as a failure state.
_shouldHaveSyncKeyBundle: false,
+ get readyToAuthenticate() {
+ // We are finished initializing when we *should* have a sync key bundle,
+ // although we might not actually have one due to auth failures etc.
+ return this._shouldHaveSyncKeyBundle;
+ },
+
get needsCustomization() {
try {
return Services.prefs.getBoolPref(PREF_SYNC_SHOW_CUSTOMIZATION);
@@ -114,34 +118,11 @@ this.BrowserIDManager.prototype = {
}
},
- hashedUID() {
- if (!this._token) {
- throw new Error("hashedUID: Don't have token");
- }
- return this._token.hashed_fxa_uid
- },
-
- deviceID() {
- return this._signedInUser && this._signedInUser.deviceId;
- },
-
initialize: function() {
for (let topic of OBSERVER_TOPICS) {
Services.obs.addObserver(this, topic, false);
}
- // and a background fetch of account data just so we can set this.account,
- // so we have a username available before we've actually done a login.
- // XXX - this is actually a hack just for tests and really shouldn't be
- // necessary. Also, you'd think it would be safe to allow this.account to
- // be set to null when there's no user logged in, but argue with the test
- // suite, not with me :)
- this._fxaService.getSignedInUser().then(accountData => {
- if (accountData) {
- this.account = accountData.email;
- }
- }).catch(err => {
- // As above, this is only for tests so it is safe to ignore.
- });
+ return this.initializeWithCurrentIdentity();
},
/**
@@ -149,7 +130,7 @@ this.BrowserIDManager.prototype = {
* the user is logged in, or is rejected if the login attempt has failed.
*/
ensureLoggedIn: function() {
- if (!this._shouldHaveSyncKeyBundle && this.whenReadyToAuthenticate) {
+ if (!this._shouldHaveSyncKeyBundle) {
// We are already in the process of logging in.
return this.whenReadyToAuthenticate.promise;
}
@@ -163,7 +144,7 @@ this.BrowserIDManager.prototype = {
// re-entering of credentials by the user is necessary we don't take any
// further action - an observer will fire when the user does that.
if (Weave.Status.login == LOGIN_FAILED_LOGIN_REJECTED) {
- return Promise.reject(new Error("User needs to re-authenticate"));
+ return Promise.reject();
}
// So - we've a previous auth problem and aren't currently attempting to
@@ -179,6 +160,7 @@ this.BrowserIDManager.prototype = {
}
this.resetCredentials();
this._signedInUser = null;
+ return Promise.resolve();
},
offerSyncOptions: function () {
@@ -202,7 +184,7 @@ this.BrowserIDManager.prototype = {
// Reset the world before we do anything async.
this.whenReadyToAuthenticate = Promise.defer();
- this.whenReadyToAuthenticate.promise.catch(err => {
+ this.whenReadyToAuthenticate.promise.then(null, (err) => {
this._log.error("Could not authenticate", err);
});
@@ -258,14 +240,14 @@ this.BrowserIDManager.prototype = {
Services.obs.notifyObservers(null, "weave:service:setup-complete", null);
Weave.Utils.nextTick(Weave.Service.sync, Weave.Service);
}
- }).catch(authErr => {
- // report what failed...
- this._log.error("Background fetch for key bundle failed", authErr);
+ }).then(null, err => {
this._shouldHaveSyncKeyBundle = true; // but we probably don't have one...
- this.whenReadyToAuthenticate.reject(authErr);
+ this.whenReadyToAuthenticate.reject(err);
+ // report what failed...
+ this._log.error("Background fetch for key bundle failed", err);
});
// and we are done - the fetch continues on in the background...
- }).catch(err => {
+ }).then(null, err => {
this._log.error("Processing logged in account", err);
});
},
@@ -301,8 +283,7 @@ this.BrowserIDManager.prototype = {
// reauth with the server - in that case we will also get here, but
// should have the same identity.
// initializeWithCurrentIdentity will throw and log if these constraints
- // aren't met (indirectly, via _updateSignedInUser()), so just go ahead
- // and do the init.
+ // aren't met, so just go ahead and do the init.
this.initializeWithCurrentIdentity(true);
break;
@@ -311,13 +292,6 @@ this.BrowserIDManager.prototype = {
// startOver will cause this instance to be thrown away, so there's
// nothing else to do.
break;
-
- case fxAccountsCommon.ON_ACCOUNT_STATE_CHANGE_NOTIFICATION:
- // throw away token and fetch a new one
- this.resetCredentials();
- this._ensureValidToken().catch(err =>
- this._log.error("Error while fetching a new token", err));
- break;
}
},
@@ -413,9 +387,6 @@ this.BrowserIDManager.prototype = {
resetCredentials: function() {
this.resetSyncKey();
this._token = null;
- // The cluster URL comes from the token, so resetting it to empty will
- // force Sync to not accidentally use a value from an earlier token.
- Weave.Service.clusterURL = null;
},
/**
@@ -503,12 +474,7 @@ this.BrowserIDManager.prototype = {
// If we still can't get keys it probably means the user authenticated
// without unlocking the MP or cleared the saved logins, so we've now
// lost them - the user will need to reauth before continuing.
- let result;
- if (this._canFetchKeys()) {
- result = STATUS_OK;
- } else {
- result = LOGIN_FAILED_LOGIN_REJECTED;
- }
+ let result = this._canFetchKeys() ? STATUS_OK : LOGIN_FAILED_LOGIN_REJECTED;
log.debug("unlockAndVerifyAuthState re-fetched credentials and is returning", result);
return result;
}
@@ -540,27 +506,14 @@ this.BrowserIDManager.prototype = {
return true;
},
- // Get our tokenServerURL - a private helper. Returns a string.
- get _tokenServerUrl() {
- // We used to support services.sync.tokenServerURI but this was a
- // pain-point for people using non-default servers as Sync may auto-reset
- // all services.sync prefs. So if that still exists, it wins.
- let url = Svc.Prefs.get("tokenServerURI"); // Svc.Prefs "root" is services.sync
- if (!url) {
- url = Services.prefs.getCharPref("identity.sync.tokenserver.uri");
- }
- while (url.endsWith("/")) { // trailing slashes cause problems...
- url = url.slice(0, -1);
- }
- return url;
- },
-
// Refresh the sync token for our user. Returns a promise that resolves
// with a token (which may be null in one sad edge-case), or rejects with an
// error.
_fetchTokenForUser: function() {
- // tokenServerURI is mis-named - convention is uri means nsISomething...
- let tokenServerURI = this._tokenServerUrl;
+ let tokenServerURI = Svc.Prefs.get("tokenServerURI");
+ if (tokenServerURI.endsWith("/")) { // trailing slashes cause problems...
+ tokenServerURI = tokenServerURI.slice(0, -1);
+ }
let log = this._log;
let client = this._tokenServerClient;
let fxa = this._fxaService;
@@ -589,7 +542,7 @@ this.BrowserIDManager.prototype = {
);
}
- let getToken = assertion => {
+ let getToken = (tokenServerURI, assertion) => {
log.debug("Getting a token");
let deferred = Promise.defer();
let cb = function (err, token) {
@@ -617,18 +570,7 @@ this.BrowserIDManager.prototype = {
return fxa.whenVerified(this._signedInUser)
.then(() => maybeFetchKeys())
.then(() => getAssertion())
- .then(assertion => getToken(assertion))
- .catch(err => {
- // If we get a 401 fetching the token it may be that our certificate
- // needs to be regenerated.
- if (!err.response || err.response.status !== 401) {
- return Promise.reject(err);
- }
- log.warn("Token server returned 401, refreshing certificate and retrying token fetch");
- return fxa.invalidateCertificate()
- .then(() => getAssertion())
- .then(assertion => getToken(assertion))
- })
+ .then(assertion => getToken(tokenServerURI, assertion))
.then(token => {
// TODO: Make it be only 80% of the duration, so refresh the token
// before it actually expires. This is to avoid sync storage errors
@@ -640,18 +582,15 @@ this.BrowserIDManager.prototype = {
}
return token;
})
- .catch(err => {
+ .then(null, err => {
// TODO: unify these errors - we need to handle errors thrown by
// both tokenserverclient and hawkclient.
// A tokenserver error thrown based on a bad response.
if (err.response && err.response.status === 401) {
- err = new AuthenticationError(err, "tokenserver");
+ err = new AuthenticationError(err);
// A hawkclient error.
} else if (err.code && err.code === 401) {
- err = new AuthenticationError(err, "hawkclient");
- // An FxAccounts.jsm error.
- } else if (err.message == fxAccountsCommon.ERROR_AUTH_ERROR) {
- err = new AuthenticationError(err, "fxaccounts");
+ err = new AuthenticationError(err);
}
// TODO: write tests to make sure that different auth error cases are handled here
@@ -673,6 +612,7 @@ this.BrowserIDManager.prototype = {
// that there is no authentication dance still under way.
this._shouldHaveSyncKeyBundle = true;
Weave.Status.login = this._authFailureReason;
+ Services.obs.notifyObservers(null, "weave:ui:login:error", null);
throw err;
});
},
@@ -684,19 +624,12 @@ this.BrowserIDManager.prototype = {
this._log.debug("_ensureValidToken already has one");
return Promise.resolve();
}
- const notifyStateChanged =
- () => Services.obs.notifyObservers(null, "weave:service:login:change", null);
// reset this._token as a safety net to reduce the possibility of us
// repeatedly attempting to use an invalid token if _fetchTokenForUser throws.
this._token = null;
return this._fetchTokenForUser().then(
token => {
this._token = token;
- notifyStateChanged();
- },
- error => {
- notifyStateChanged();
- throw error
}
);
},
@@ -719,16 +652,9 @@ this.BrowserIDManager.prototype = {
_getAuthenticationHeader: function(httpObject, method) {
let cb = Async.makeSpinningCallback();
this._ensureValidToken().then(cb, cb);
- // Note that in failure states we return null, causing the request to be
- // made without authorization headers, thereby presumably causing a 401,
- // which causes Sync to log out. If we throw, this may not happen as
- // expected.
try {
cb.wait();
} catch (ex) {
- if (Async.isShutdownException(ex)) {
- throw ex;
- }
this._log.error("Failed to fetch a token for authentication", ex);
return null;
}
@@ -764,17 +690,8 @@ this.BrowserIDManager.prototype = {
createClusterManager: function(service) {
return new BrowserIDClusterManager(service);
- },
+ }
- // Tell Sync what the login status should be if it saw a 401 fetching
- // info/collections as part of login verification (typically immediately
- // after login.)
- // In our case, it almost certainly means a transient error fetching a token
- // (and hitting this will cause us to logout, which will correctly handle an
- // authoritative login issue.)
- loginStatusFromVerification404() {
- return LOGIN_FAILED_NETWORK_ERROR;
- },
};
/* An implementation of the ClusterManager for this identity
@@ -820,7 +737,7 @@ BrowserIDClusterManager.prototype = {
// it's likely a 401 was received using the existing token - in which
// case we just discard the existing token and fetch a new one.
if (this.service.clusterURL) {
- log.debug("_findCluster has a pre-existing clusterURL, so discarding the current token");
+ log.debug("_findCluster found existing clusterURL, so discarding the current token");
this.identity._token = null;
}
return this.identity._ensureValidToken();
diff --git a/services/sync/modules/collection_validator.js b/services/sync/modules/collection_validator.js
deleted file mode 100644
index 41141bba3..000000000
--- a/services/sync/modules/collection_validator.js
+++ /dev/null
@@ -1,204 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this file,
- * You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-"use strict";
-
-const Cu = Components.utils;
-
-Cu.import("resource://services-sync/record.js");
-Cu.import("resource://services-sync/main.js");
-
-this.EXPORTED_SYMBOLS = ["CollectionValidator", "CollectionProblemData"];
-
-class CollectionProblemData {
- constructor() {
- this.missingIDs = 0;
- this.duplicates = [];
- this.clientMissing = [];
- this.serverMissing = [];
- this.serverDeleted = [];
- this.serverUnexpected = [];
- this.differences = [];
- }
-
- /**
- * Produce a list summarizing problems found. Each entry contains {name, count},
- * where name is the field name for the problem, and count is the number of times
- * the problem was encountered.
- *
- * Validation has failed if all counts are not 0.
- */
- getSummary() {
- return [
- { name: "clientMissing", count: this.clientMissing.length },
- { name: "serverMissing", count: this.serverMissing.length },
- { name: "serverDeleted", count: this.serverDeleted.length },
- { name: "serverUnexpected", count: this.serverUnexpected.length },
- { name: "differences", count: this.differences.length },
- { name: "missingIDs", count: this.missingIDs },
- { name: "duplicates", count: this.duplicates.length }
- ];
- }
-}
-
-class CollectionValidator {
- // Construct a generic collection validator. This is intended to be called by
- // subclasses.
- // - name: Name of the engine
- // - idProp: Property that identifies a record. That is, if a client and server
- // record have the same value for the idProp property, they should be
- // compared against eachother.
- // - props: Array of properties that should be compared
- constructor(name, idProp, props) {
- this.name = name;
- this.props = props;
- this.idProp = idProp;
- }
-
- // Should a custom ProblemData type be needed, return it here.
- emptyProblemData() {
- return new CollectionProblemData();
- }
-
- getServerItems(engine) {
- let collection = engine.itemSource();
- let collectionKey = engine.service.collectionKeys.keyForCollection(engine.name);
- collection.full = true;
- let items = [];
- collection.recordHandler = function(item) {
- item.decrypt(collectionKey);
- items.push(item.cleartext);
- };
- let resp = collection.getBatched();
- if (!resp.success) {
- throw resp;
- }
- return items;
- }
-
- // Should return a promise that resolves to an array of client items.
- getClientItems() {
- return Promise.reject("Must implement");
- }
-
- // Turn the client item into something that can be compared with the server item,
- // and is also safe to mutate.
- normalizeClientItem(item) {
- return Cu.cloneInto(item, {});
- }
-
- // Turn the server item into something that can be easily compared with the client
- // items.
- normalizeServerItem(item) {
- return item;
- }
-
- // Return whether or not a server item should be present on the client. Expected
- // to be overridden.
- clientUnderstands(item) {
- return true;
- }
-
- // Return whether or not a client item should be present on the server. Expected
- // to be overridden
- syncedByClient(item) {
- return true;
- }
-
- // Compare the server item and the client item, and return a list of property
- // names that are different. Can be overridden if needed.
- getDifferences(client, server) {
- let differences = [];
- for (let prop of this.props) {
- let clientProp = client[prop];
- let serverProp = server[prop];
- if ((clientProp || "") !== (serverProp || "")) {
- differences.push(prop);
- }
- }
- return differences;
- }
-
- // Returns an object containing
- // problemData: an instance of the class returned by emptyProblemData(),
- // clientRecords: Normalized client records
- // records: Normalized server records,
- // deletedRecords: Array of ids that were marked as deleted by the server.
- compareClientWithServer(clientItems, serverItems) {
- clientItems = clientItems.map(item => this.normalizeClientItem(item));
- serverItems = serverItems.map(item => this.normalizeServerItem(item));
- let problems = this.emptyProblemData();
- let seenServer = new Map();
- let serverDeleted = new Set();
- let allRecords = new Map();
-
- for (let record of serverItems) {
- let id = record[this.idProp];
- if (!id) {
- ++problems.missingIDs;
- continue;
- }
- if (record.deleted) {
- serverDeleted.add(record);
- } else {
- let possibleDupe = seenServer.get(id);
- if (possibleDupe) {
- problems.duplicates.push(id);
- } else {
- seenServer.set(id, record);
- allRecords.set(id, { server: record, client: null, });
- }
- record.understood = this.clientUnderstands(record);
- }
- }
-
- let recordPairs = [];
- let seenClient = new Map();
- for (let record of clientItems) {
- let id = record[this.idProp];
- record.shouldSync = this.syncedByClient(record);
- seenClient.set(id, record);
- let combined = allRecords.get(id);
- if (combined) {
- combined.client = record;
- } else {
- allRecords.set(id, { client: record, server: null });
- }
- }
-
- for (let [id, { server, client }] of allRecords) {
- if (!client && !server) {
- throw new Error("Impossible: no client or server record for " + id);
- } else if (server && !client) {
- if (server.understood) {
- problems.clientMissing.push(id);
- }
- } else if (client && !server) {
- if (client.shouldSync) {
- problems.serverMissing.push(id);
- }
- } else {
- if (!client.shouldSync) {
- if (!problems.serverUnexpected.includes(id)) {
- problems.serverUnexpected.push(id);
- }
- continue;
- }
- let differences = this.getDifferences(client, server);
- if (differences && differences.length) {
- problems.differences.push({ id, differences });
- }
- }
- }
- return {
- problemData: problems,
- clientRecords: clientItems,
- records: serverItems,
- deletedRecords: [...serverDeleted]
- };
- }
-}
-
-// Default to 0, some engines may override.
-CollectionValidator.prototype.version = 0;
diff --git a/services/sync/modules/constants.js b/services/sync/modules/constants.js
index f70bbd61c..c8d66d921 100644
--- a/services/sync/modules/constants.js
+++ b/services/sync/modules/constants.js
@@ -4,8 +4,7 @@
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
// Process each item in the "constants hash" to add to "global" and give a name
-this.EXPORTED_SYMBOLS = [];
-for (let [key, val] of Object.entries({
+this.EXPORTED_SYMBOLS = [((this[key] = val), key) for ([key, val] in Iterator({
WEAVE_VERSION: "@weave_version@",
@@ -45,7 +44,7 @@ MAX_IGNORE_ERROR_COUNT: 5,
// Backoff intervals
MINIMUM_BACKOFF_INTERVAL: 15 * 60 * 1000, // 15 minutes
-MAXIMUM_BACKOFF_INTERVAL: 8 * 60 * 60 * 1000, // 8 hours
+MAXIMUM_BACKOFF_INTERVAL: 8 * 60 * 60 * 1000, // 8 hours
// HMAC event handling timeout.
// 10 minutes: a compromise between the multi-desktop sync interval
@@ -76,10 +75,6 @@ PASSWORDS_STORE_BATCH_SIZE: 50, // same as MOBILE_BATCH_SIZE
ADDONS_STORE_BATCH_SIZE: 1000000, // process all addons at once
APPS_STORE_BATCH_SIZE: 50, // same as MOBILE_BATCH_SIZE
-// Default batch size for download batching
-// (how many records are fetched at a time from the server when batching is used).
-DEFAULT_DOWNLOAD_BATCH_SIZE: 1000,
-
// score thresholds for early syncs
SINGLE_USER_THRESHOLD: 1000,
MULTI_DEVICE_THRESHOLD: 300,
@@ -98,16 +93,13 @@ SCORE_UPDATE_DELAY: 100,
// observed spurious idle/back events and short enough to pre-empt user activity.
IDLE_OBSERVER_BACK_DELAY: 100,
-// Max number of records or bytes to upload in a single POST - we'll do multiple POSTS if either
-// MAX_UPLOAD_RECORDS or MAX_UPLOAD_BYTES is hit)
+// Number of records to upload in a single POST (multiple POSTS if exceeded)
+// FIXME: Record size limit is 256k (new cluster), so this can be quite large!
+// (Bug 569295)
MAX_UPLOAD_RECORDS: 100,
-MAX_UPLOAD_BYTES: 1024 * 1023, // just under 1MB
MAX_HISTORY_UPLOAD: 5000,
MAX_HISTORY_DOWNLOAD: 5000,
-// TTL of the message sent to another device when sending a tab
-NOTIFY_TAB_SENT_TTL_SECS: 1 * 3600, // 1 hour
-
// Top-level statuses:
STATUS_OK: "success.status_ok",
SYNC_FAILED: "error.sync.failed",
@@ -130,6 +122,7 @@ LOGIN_FAILED_NETWORK_ERROR: "error.login.reason.network",
LOGIN_FAILED_SERVER_ERROR: "error.login.reason.server",
LOGIN_FAILED_INVALID_PASSPHRASE: "error.login.reason.recoverykey",
LOGIN_FAILED_LOGIN_REJECTED: "error.login.reason.account",
+LOGIN_FAILED_NOT_READY: "error.login.reason.initializing",
// sync failure status codes
METARECORD_DOWNLOAD_FAIL: "error.sync.reason.metarecord_download_fail",
@@ -152,8 +145,6 @@ ENGINE_UNKNOWN_FAIL: "error.engine.reason.unknown_fail",
ENGINE_APPLY_FAIL: "error.engine.reason.apply_fail",
ENGINE_METARECORD_DOWNLOAD_FAIL: "error.engine.reason.metarecord_download_fail",
ENGINE_METARECORD_UPLOAD_FAIL: "error.engine.reason.metarecord_upload_fail",
-// an upload failure where the batch was interrupted with a 412
-ENGINE_BATCH_INTERRUPTED: "error.engine.reason.batch_interrupted",
JPAKE_ERROR_CHANNEL: "jpake.error.channel",
JPAKE_ERROR_NETWORK: "jpake.error.network",
@@ -181,7 +172,7 @@ kSyncBackoffNotMet: "Trying to sync before the server said it
kFirstSyncChoiceNotMade: "User has not selected an action for first sync",
// Application IDs
-FIREFOX_ID: "{ec8030f7-c20a-464f-9b0e-13a3a9e97384}",
+FIREFOX_ID: "{8de7fcbb-c55c-4fbe-bfc5-fc555c87dbc4}",
FENNEC_ID: "{a23983c0-fd0e-11dc-95ff-0800200c9a66}",
SEAMONKEY_ID: "{92650c4d-4b8e-4d2a-b7eb-24ecf4f6b63a}",
TEST_HARNESS_ID: "xuth@mozilla.org",
@@ -189,10 +180,6 @@ TEST_HARNESS_ID: "xuth@mozilla.org",
MIN_PP_LENGTH: 12,
MIN_PASS_LENGTH: 8,
-DEVICE_TYPE_DESKTOP: "desktop",
-DEVICE_TYPE_MOBILE: "mobile",
+LOG_DATE_FORMAT: "%Y-%m-%d %H:%M:%S",
-})) {
- this[key] = val;
- this.EXPORTED_SYMBOLS.push(key);
-}
+}))];
diff --git a/services/sync/modules/engines.js b/services/sync/modules/engines.js
index 1eaa1863a..eabddae2b 100644
--- a/services/sync/modules/engines.js
+++ b/services/sync/modules/engines.js
@@ -7,24 +7,21 @@ this.EXPORTED_SYMBOLS = [
"Engine",
"SyncEngine",
"Tracker",
- "Store",
- "Changeset"
+ "Store"
];
-var {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components;
+const {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components;
Cu.import("resource://services-common/async.js");
Cu.import("resource://gre/modules/Log.jsm");
Cu.import("resource://services-common/observers.js");
+Cu.import("resource://services-common/utils.js");
Cu.import("resource://services-sync/constants.js");
Cu.import("resource://services-sync/identity.js");
Cu.import("resource://services-sync/record.js");
Cu.import("resource://services-sync/resource.js");
Cu.import("resource://services-sync/util.js");
-XPCOMUtils.defineLazyModuleGetter(this, "fxAccounts",
- "resource://gre/modules/FxAccounts.jsm");
-
/*
* Trackers are associated with a single engine and deal with
* listening for changes to their particular data type.
@@ -107,7 +104,7 @@ Tracker.prototype = {
Utils.jsonLoad("changes/" + this.file, this, function(json) {
if (json && (typeof(json) == "object")) {
this.changedIDs = json;
- } else if (json !== null) {
+ } else {
this._log.warn("Changed IDs file " + this.file + " contains non-object value.");
json = null;
}
@@ -132,30 +129,26 @@ Tracker.prototype = {
this._ignored.splice(index, 1);
},
- _saveChangedID(id, when) {
- this._log.trace(`Adding changed ID: ${id}, ${JSON.stringify(when)}`);
- this.changedIDs[id] = when;
- this.saveChangedIDs(this.onSavedChangedIDs);
- },
-
addChangedID: function (id, when) {
if (!id) {
this._log.warn("Attempted to add undefined ID to tracker");
return false;
}
- if (this.ignoreAll || this._ignored.includes(id)) {
+ if (this.ignoreAll || (id in this._ignored)) {
return false;
}
// Default to the current time in seconds if no time is provided.
if (when == null) {
- when = this._now();
+ when = Math.floor(Date.now() / 1000);
}
// Add/update the entry if we have a newer time.
if ((this.changedIDs[id] || -Infinity) < when) {
- this._saveChangedID(id, when);
+ this._log.trace("Adding changed ID: " + id + ", " + when);
+ this.changedIDs[id] = when;
+ this.saveChangedIDs(this.onSavedChangedIDs);
}
return true;
@@ -166,9 +159,8 @@ Tracker.prototype = {
this._log.warn("Attempted to remove undefined ID to tracker");
return false;
}
- if (this.ignoreAll || this._ignored.includes(id)) {
+ if (this.ignoreAll || (id in this._ignored))
return false;
- }
if (this.changedIDs[id] != null) {
this._log.trace("Removing changed ID " + id);
delete this.changedIDs[id];
@@ -183,10 +175,6 @@ Tracker.prototype = {
this.saveChangedIDs();
},
- _now() {
- return Date.now() / 1000;
- },
-
_isTracking: false,
// Override these in your subclasses.
@@ -311,21 +299,17 @@ Store.prototype = {
*/
applyIncomingBatch: function (records) {
let failed = [];
- for (let record of records) {
+ for each (let record in records) {
try {
this.applyIncoming(record);
+ } catch (ex if (ex.code == Engine.prototype.eEngineAbortApplyIncoming)) {
+ // This kind of exception should have a 'cause' attribute, which is an
+ // originating exception.
+ // ex.cause will carry its stack with it when rethrown.
+ throw ex.cause;
} catch (ex) {
- if (ex.code == Engine.prototype.eEngineAbortApplyIncoming) {
- // This kind of exception should have a 'cause' attribute, which is an
- // originating exception.
- // ex.cause will carry its stack with it when rethrown.
- throw ex.cause;
- }
- if (Async.isShutdownException(ex)) {
- throw ex;
- }
- this._log.warn("Failed to apply incoming record " + record.id, ex);
- this.engine._noteApplyFailure();
+ this._log.warn("Failed to apply incoming record " + record.id);
+ this._log.warn("Encountered exception: " + Utils.exceptionStr(ex));
failed.push(record.id);
}
};
@@ -499,11 +483,7 @@ EngineManager.prototype = {
},
getAll: function () {
- let engines = [];
- for (let [, engine] of Object.entries(this._engines)) {
- engines.push(engine);
- }
- return engines;
+ return [engine for ([name, engine] in Iterator(this._engines))];
},
/**
@@ -516,7 +496,7 @@ EngineManager.prototype = {
},
get enabledEngineNames() {
- return this.getEnabled().map(e => e.name);
+ return [e.name for each (e in this.getEnabled())];
},
persistDeclined: function () {
@@ -593,11 +573,16 @@ EngineManager.prototype = {
this._engines[name] = engine;
}
} catch (ex) {
+ this._log.error(CommonUtils.exceptionStr(ex));
+
+ let mesg = ex.message ? ex.message : ex;
let name = engineObject || "";
name = name.prototype || "";
name = name.name || "";
- this._log.error(`Could not initialize engine ${name}`, ex);
+ let out = "Could not initialize engine '" + name + "': " + mesg;
+ this._log.error(out);
+
return engineObject;
}
},
@@ -643,17 +628,16 @@ Engine.prototype = {
// Signal to the engine that processing further records is pointless.
eEngineAbortApplyIncoming: "error.engine.abort.applyincoming",
- // Should we keep syncing if we find a record that cannot be uploaded (ever)?
- // If this is false, we'll throw, otherwise, we'll ignore the record and
- // continue. This currently can only happen due to the record being larger
- // than the record upload limit.
- allowSkippedRecord: true,
-
get prefName() {
return this.name;
},
get enabled() {
+ // XXX: Disable non-functional add-ons syncing for the time being
+ // This check can go away when add-on syncing is addressed
+ if (this.prefName == "addons")
+ return false;
+
return Svc.Prefs.get("engine." + this.prefName, false);
},
@@ -711,15 +695,6 @@ Engine.prototype = {
wipeClient: function () {
this._notify("wipe-client", this.name, this._wipeClient)();
- },
-
- /**
- * If one exists, initialize and return a validator for this engine (which
- * must have a `validate(engine)` method that returns a promise to an object
- * with a getSummary method). Otherwise return null.
- */
- getValidator: function () {
- return null;
}
};
@@ -813,11 +788,7 @@ SyncEngine.prototype = {
return this._toFetch;
},
set toFetch(val) {
- let cb = (error) => {
- if (error) {
- this._log.error("Failed to read JSON records to fetch", error);
- }
- }
+ let cb = (error) => this._log.error(Utils.exceptionStr(error));
// Coerce the array to a string for more efficient comparison.
if (val + "" == this._toFetch) {
return;
@@ -842,13 +813,7 @@ SyncEngine.prototype = {
return this._previousFailed;
},
set previousFailed(val) {
- let cb = (error) => {
- if (error) {
- this._log.error("Failed to set previousFailed", error);
- } else {
- this._log.debug("Successfully wrote previousFailed.");
- }
- }
+ let cb = (error) => this._log.error(Utils.exceptionStr(error));
// Coerce the array to a string for more efficient comparison.
if (val + "" == this._previousFailed) {
return;
@@ -881,8 +846,9 @@ SyncEngine.prototype = {
},
/*
- * Returns a changeset for this sync. Engine implementations can override this
- * method to bypass the tracker for certain or all changed items.
+ * Returns a mapping of IDs -> changed timestamp. Engine implementations
+ * can override this method to bypass the tracker for certain or all
+ * changed items.
*/
getChangedIDs: function () {
return this._tracker.changedIDs;
@@ -950,16 +916,20 @@ SyncEngine.prototype = {
// this._modified to the tracker.
this.lastSyncLocal = Date.now();
if (this.lastSync) {
- this._modified = this.pullNewChanges();
+ this._modified = this.getChangedIDs();
} else {
+ // Mark all items to be uploaded, but treat them as changed from long ago
this._log.debug("First sync, uploading all items");
- this._modified = this.pullAllChanges();
+ this._modified = {};
+ for (let id in this._store.getAllIDs()) {
+ this._modified[id] = 0;
+ }
}
// Clear the tracker now. If the sync fails we'll add the ones we failed
// to upload back.
this._tracker.clearChangedIDs();
- this._log.info(this._modified.count() +
+ this._log.info(Object.keys(this._modified).length +
" outgoing items pre-reconciliation");
// Keep track of what to delete at the end of sync
@@ -970,7 +940,7 @@ SyncEngine.prototype = {
* A tiny abstraction to make it easier to test incoming record
* application.
*/
- itemSource: function () {
+ _itemSource: function () {
return new Collection(this.engineURL, this._recordObj, this.service);
},
@@ -987,7 +957,7 @@ SyncEngine.prototype = {
let isMobile = (Svc.Prefs.get("client.type") == "mobile");
if (!newitems) {
- newitems = this.itemSource();
+ newitems = this._itemSource();
}
if (this._defaultSort) {
@@ -1024,12 +994,10 @@ SyncEngine.prototype = {
try {
failed = failed.concat(this._store.applyIncomingBatch(applyBatch));
} catch (ex) {
- if (Async.isShutdownException(ex)) {
- throw ex;
- }
// Catch any error that escapes from applyIncomingBatch. At present
// those will all be abort events.
- this._log.warn("Got exception, aborting processIncoming", ex);
+ this._log.warn("Got exception " + Utils.exceptionStr(ex) +
+ ", aborting processIncoming.");
aborting = ex;
}
this._tracker.ignoreAll = false;
@@ -1074,10 +1042,7 @@ SyncEngine.prototype = {
try {
try {
item.decrypt(key);
- } catch (ex) {
- if (!Utils.isHMACMismatch(ex)) {
- throw ex;
- }
+ } catch (ex if Utils.isHMACMismatch(ex)) {
let strategy = self.handleHMACMismatch(item, true);
if (strategy == SyncEngine.kRecoveryStrategy.retry) {
// You only get one retry.
@@ -1087,10 +1052,7 @@ SyncEngine.prototype = {
key = self.service.collectionKeys.keyForCollection(self.name);
item.decrypt(key);
strategy = null;
- } catch (ex) {
- if (!Utils.isHMACMismatch(ex)) {
- throw ex;
- }
+ } catch (ex if Utils.isHMACMismatch(ex)) {
strategy = self.handleHMACMismatch(item, false);
}
}
@@ -1103,8 +1065,7 @@ SyncEngine.prototype = {
self._log.debug("Ignoring second retry suggestion.");
// Fall through to error case.
case SyncEngine.kRecoveryStrategy.error:
- self._log.warn("Error decrypting record", ex);
- self._noteApplyFailure();
+ self._log.warn("Error decrypting record: " + Utils.exceptionStr(ex));
failed.push(item.id);
return;
case SyncEngine.kRecoveryStrategy.ignore:
@@ -1114,11 +1075,7 @@ SyncEngine.prototype = {
}
}
} catch (ex) {
- if (Async.isShutdownException(ex)) {
- throw ex;
- }
- self._log.warn("Error decrypting record", ex);
- self._noteApplyFailure();
+ self._log.warn("Error decrypting record: " + Utils.exceptionStr(ex));
failed.push(item.id);
return;
}
@@ -1126,20 +1083,15 @@ SyncEngine.prototype = {
let shouldApply;
try {
shouldApply = self._reconcile(item);
+ } catch (ex if (ex.code == Engine.prototype.eEngineAbortApplyIncoming)) {
+ self._log.warn("Reconciliation failed: aborting incoming processing.");
+ failed.push(item.id);
+ aborting = ex.cause;
} catch (ex) {
- if (ex.code == Engine.prototype.eEngineAbortApplyIncoming) {
- self._log.warn("Reconciliation failed: aborting incoming processing.");
- self._noteApplyFailure();
- failed.push(item.id);
- aborting = ex.cause;
- } else if (!Async.isShutdownException(ex)) {
- self._log.warn("Failed to reconcile incoming record " + item.id, ex);
- self._noteApplyFailure();
- failed.push(item.id);
- return;
- } else {
- throw ex;
- }
+ self._log.warn("Failed to reconcile incoming record " + item.id);
+ self._log.warn("Encountered exception: " + Utils.exceptionStr(ex));
+ failed.push(item.id);
+ return;
}
if (shouldApply) {
@@ -1158,7 +1110,7 @@ SyncEngine.prototype = {
// Only bother getting data from the server if there's new things
if (this.lastModified == null || this.lastModified > this.lastSync) {
- let resp = newitems.getBatched();
+ let resp = newitems.get();
doApplyBatchAndPersistFailed.call(this);
if (!resp.success) {
resp.failureCode = ENGINE_DOWNLOAD_FAIL;
@@ -1243,13 +1195,7 @@ SyncEngine.prototype = {
// Apply remaining items.
doApplyBatchAndPersistFailed.call(this);
- count.newFailed = this.previousFailed.reduce((count, engine) => {
- if (failedInPreviousSync.indexOf(engine) == -1) {
- count++;
- this._noteApplyNewFailure();
- }
- return count;
- }, 0);
+ count.newFailed = Utils.arraySub(this.previousFailed, failedInPreviousSync).length;
count.succeeded = Math.max(0, count.applied - count.failed);
this._log.info(["Records:",
count.applied, "applied,",
@@ -1260,14 +1206,6 @@ SyncEngine.prototype = {
Observers.notify("weave:engine:sync:applied", count, this.name);
},
- _noteApplyFailure: function () {
- // here would be a good place to record telemetry...
- },
-
- _noteApplyNewFailure: function () {
- // here would be a good place to record telemetry...
- },
-
/**
* Find a GUID of an item that is a duplicate of the incoming item but happens
* to have a different GUID
@@ -1278,16 +1216,6 @@ SyncEngine.prototype = {
// By default, assume there's no dupe items for the engine
},
- // Called when the server has a record marked as deleted, but locally we've
- // changed it more recently than the deletion. If we return false, the
- // record will be deleted locally. If we return true, we'll reupload the
- // record to the server -- any extra work that's needed as part of this
- // process should be done at this point (such as mark the record's parent
- // for reuploading in the case of bookmarks).
- _shouldReviveRemotelyDeletedRecord(remoteItem) {
- return true;
- },
-
_deleteId: function (id) {
this._tracker.removeChangedID(id);
@@ -1298,18 +1226,6 @@ SyncEngine.prototype = {
this._delete.ids.push(id);
},
- _switchItemToDupe(localDupeGUID, incomingItem) {
- // The local, duplicate ID is always deleted on the server.
- this._deleteId(localDupeGUID);
-
- // We unconditionally change the item's ID in case the engine knows of
- // an item but doesn't expose it through itemExists. If the API
- // contract were stronger, this could be changed.
- this._log.debug("Switching local ID to incoming: " + localDupeGUID + " -> " +
- incomingItem.id);
- this._store.changeItemID(localDupeGUID, incomingItem.id);
- },
-
/**
* Reconcile incoming record with local state.
*
@@ -1329,12 +1245,12 @@ SyncEngine.prototype = {
// because some state may change during the course of this function and we
// need to operate on the original values.
let existsLocally = this._store.itemExists(item.id);
- let locallyModified = this._modified.has(item.id);
+ let locallyModified = item.id in this._modified;
// TODO Handle clock drift better. Tracked in bug 721181.
let remoteAge = AsyncResource.serverTime - item.modified;
let localAge = locallyModified ?
- (Date.now() / 1000 - this._modified.getModifiedTimestamp(item.id)) : null;
+ (Date.now() / 1000 - this._modified[item.id]) : null;
let remoteIsNewer = remoteAge < localAge;
this._log.trace("Reconciling " + item.id + ". exists=" +
@@ -1363,18 +1279,15 @@ SyncEngine.prototype = {
"exists and isn't modified.");
return true;
}
- this._log.trace("Incoming record is deleted but we had local changes.");
- if (remoteIsNewer) {
- this._log.trace("Remote record is newer -- deleting local record.");
- return true;
- }
- // If the local record is newer, we defer to individual engines for
- // how to handle this. By default, we revive the record.
- let willRevive = this._shouldReviveRemotelyDeletedRecord(item);
- this._log.trace("Local record is newer -- reviving? " + willRevive);
-
- return !willRevive;
+ // TODO As part of bug 720592, determine whether we should do more here.
+ // In the case where the local changes are newer, it is quite possible
+ // that the local client will restore data a remote client had tried to
+ // delete. There might be a good reason for that delete and it might be
+ // enexpected for this client to restore that data.
+ this._log.trace("Incoming record is deleted but we had local changes. " +
+ "Applying the youngest record.");
+ return remoteIsNewer;
}
// At this point the incoming record is not for a deletion and must have
@@ -1386,32 +1299,40 @@ SyncEngine.prototype = {
// refresh the metadata collected above. See bug 710448 for the history
// of this logic.
if (!existsLocally) {
- let localDupeGUID = this._findDupe(item);
- if (localDupeGUID) {
- this._log.trace("Local item " + localDupeGUID + " is a duplicate for " +
+ let dupeID = this._findDupe(item);
+ if (dupeID) {
+ this._log.trace("Local item " + dupeID + " is a duplicate for " +
"incoming item " + item.id);
+ // The local, duplicate ID is always deleted on the server.
+ this._deleteId(dupeID);
+
// The current API contract does not mandate that the ID returned by
// _findDupe() actually exists. Therefore, we have to perform this
// check.
- existsLocally = this._store.itemExists(localDupeGUID);
+ existsLocally = this._store.itemExists(dupeID);
+
+ // We unconditionally change the item's ID in case the engine knows of
+ // an item but doesn't expose it through itemExists. If the API
+ // contract were stronger, this could be changed.
+ this._log.debug("Switching local ID to incoming: " + dupeID + " -> " +
+ item.id);
+ this._store.changeItemID(dupeID, item.id);
// If the local item was modified, we carry its metadata forward so
// appropriate reconciling can be performed.
- if (this._modified.has(localDupeGUID)) {
+ if (dupeID in this._modified) {
locallyModified = true;
- localAge = this._tracker._now() - this._modified.getModifiedTimestamp(localDupeGUID);
+ localAge = Date.now() / 1000 - this._modified[dupeID];
remoteIsNewer = remoteAge < localAge;
- this._modified.swap(localDupeGUID, item.id);
+ this._modified[item.id] = this._modified[dupeID];
+ delete this._modified[dupeID];
} else {
locallyModified = false;
localAge = null;
}
- // Tell the engine to do whatever it needs to switch the items.
- this._switchItemToDupe(localDupeGUID, item);
-
this._log.debug("Local item after duplication: age=" + localAge +
"; modified=" + locallyModified + "; exists=" +
existsLocally);
@@ -1440,7 +1361,7 @@ SyncEngine.prototype = {
if (remoteIsNewer) {
this._log.trace("Applying incoming because local item was deleted " +
"before the incoming item was changed.");
- this._modified.delete(item.id);
+ delete this._modified[item.id];
return true;
}
@@ -1466,7 +1387,7 @@ SyncEngine.prototype = {
this._log.trace("Ignoring incoming item because the local item is " +
"identical.");
- this._modified.delete(item.id);
+ delete this._modified[item.id];
return false;
}
@@ -1491,97 +1412,69 @@ SyncEngine.prototype = {
_uploadOutgoing: function () {
this._log.trace("Uploading local changes to server.");
- let modifiedIDs = this._modified.ids();
+ let modifiedIDs = Object.keys(this._modified);
if (modifiedIDs.length) {
this._log.trace("Preparing " + modifiedIDs.length +
" outgoing records");
- let counts = { sent: modifiedIDs.length, failed: 0 };
-
// collection we'll upload
let up = new Collection(this.engineURL, null, this.service);
+ let count = 0;
- let failed = [];
- let successful = [];
- let handleResponse = (resp, batchOngoing = false) => {
- // Note: We don't want to update this.lastSync, or this._modified until
- // the batch is complete, however we want to remember success/failure
- // indicators for when that happens.
+ // Upload what we've got so far in the collection
+ let doUpload = Utils.bind2(this, function(desc) {
+ this._log.info("Uploading " + desc + " of " + modifiedIDs.length +
+ " records");
+ let resp = up.post();
if (!resp.success) {
this._log.debug("Uploading records failed: " + resp);
- resp.failureCode = resp.status == 412 ? ENGINE_BATCH_INTERRUPTED : ENGINE_UPLOAD_FAIL;
+ resp.failureCode = ENGINE_UPLOAD_FAIL;
throw resp;
}
// Update server timestamp from the upload.
- failed = failed.concat(Object.keys(resp.obj.failed));
- successful = successful.concat(resp.obj.success);
-
- if (batchOngoing) {
- // Nothing to do yet
- return;
- }
- // Advance lastSync since we've finished the batch.
let modified = resp.headers["x-weave-timestamp"];
- if (modified > this.lastSync) {
+ if (modified > this.lastSync)
this.lastSync = modified;
- }
- if (failed.length && this._log.level <= Log.Level.Debug) {
+
+ let failed_ids = Object.keys(resp.obj.failed);
+ if (failed_ids.length)
this._log.debug("Records that will be uploaded again because "
+ "the server couldn't store them: "
- + failed.join(", "));
- }
-
- counts.failed += failed.length;
+ + failed_ids.join(", "));
- for (let id of successful) {
- this._modified.delete(id);
+ // Clear successfully uploaded objects.
+ for each (let id in resp.obj.success) {
+ delete this._modified[id];
}
- this._onRecordsWritten(successful, failed);
-
- // clear for next batch
- failed.length = 0;
- successful.length = 0;
- };
-
- let postQueue = up.newPostQueue(this._log, this.lastSync, handleResponse);
+ up.clearRecords();
+ });
- for (let id of modifiedIDs) {
- let out;
- let ok = false;
+ for each (let id in modifiedIDs) {
try {
- out = this._createRecord(id);
+ let out = this._createRecord(id);
if (this._log.level <= Log.Level.Trace)
this._log.trace("Outgoing: " + out);
out.encrypt(this.service.collectionKeys.keyForCollection(this.name));
- ok = true;
- } catch (ex) {
- if (Async.isShutdownException(ex)) {
- throw ex;
- }
- this._log.warn("Error creating record", ex);
+ up.pushData(out);
}
- if (ok) {
- let { enqueued, error } = postQueue.enqueue(out);
- if (!enqueued) {
- ++counts.failed;
- if (!this.allowSkippedRecord) {
- throw error;
- }
- }
+ catch(ex) {
+ this._log.warn("Error creating record: " + Utils.exceptionStr(ex));
}
+
+ // Partial upload
+ if ((++count % MAX_UPLOAD_RECORDS) == 0)
+ doUpload((count - MAX_UPLOAD_RECORDS) + " - " + count + " out");
+
this._store._sleep(0);
}
- postQueue.flush(true);
- Observers.notify("weave:engine:sync:uploaded", counts, this.name);
- }
- },
- _onRecordsWritten(succeeded, failed) {
- // Implement this method to take specific actions against successfully
- // uploaded records and failed records.
+ // Final upload
+ if (count % MAX_UPLOAD_RECORDS > 0)
+ doUpload(count >= MAX_UPLOAD_RECORDS ? "last batch" : "all");
+ }
},
// Any cleanup necessary.
@@ -1596,7 +1489,7 @@ SyncEngine.prototype = {
coll.delete();
});
- for (let [key, val] of Object.entries(this._delete)) {
+ for (let [key, val] in Iterator(this._delete)) {
// Remove the key for future uses
delete this._delete[key];
@@ -1619,8 +1512,10 @@ SyncEngine.prototype = {
}
// Mark failed WBOs as changed again so they are reuploaded next time.
- this.trackRemainingChanges();
- this._modified.clear();
+ for (let [id, when] in Iterator(this._modified)) {
+ this._tracker.addChangedID(id, when);
+ }
+ this._modified = {};
},
_sync: function () {
@@ -1656,11 +1551,9 @@ SyncEngine.prototype = {
try {
this._log.trace("Trying to decrypt a record from the server..");
test.get();
- } catch (ex) {
- if (Async.isShutdownException(ex)) {
- throw ex;
- }
- this._log.debug("Failed test decrypt", ex);
+ }
+ catch(ex) {
+ this._log.debug("Failed test decrypt: " + Utils.exceptionStr(ex));
}
return canDecrypt;
@@ -1706,108 +1599,5 @@ SyncEngine.prototype = {
return (this.service.handleHMACEvent() && mayRetry) ?
SyncEngine.kRecoveryStrategy.retry :
SyncEngine.kRecoveryStrategy.error;
- },
-
- /**
- * Returns a changeset containing all items in the store. The default
- * implementation returns a changeset with timestamps from long ago, to
- * ensure we always use the remote version if one exists.
- *
- * This function is only called for the first sync. Subsequent syncs call
- * `pullNewChanges`.
- *
- * @return A `Changeset` object.
- */
- pullAllChanges() {
- let changeset = new Changeset();
- for (let id in this._store.getAllIDs()) {
- changeset.set(id, 0);
- }
- return changeset;
- },
-
- /*
- * Returns a changeset containing entries for all currently tracked items.
- * The default implementation returns a changeset with timestamps indicating
- * when the item was added to the tracker.
- *
- * @return A `Changeset` object.
- */
- pullNewChanges() {
- return new Changeset(this.getChangedIDs());
- },
-
- /**
- * Adds all remaining changeset entries back to the tracker, typically for
- * items that failed to upload. This method is called at the end of each sync.
- *
- */
- trackRemainingChanges() {
- for (let [id, change] of this._modified.entries()) {
- this._tracker.addChangedID(id, change);
- }
- },
-};
-
-/**
- * A changeset is created for each sync in `Engine::get{Changed, All}IDs`,
- * and stores opaque change data for tracked IDs. The default implementation
- * only records timestamps, though engines can extend this to store additional
- * data for each entry.
- */
-class Changeset {
- // Creates a changeset with an initial set of tracked entries.
- constructor(changes = {}) {
- this.changes = changes;
- }
-
- // Returns the last modified time, in seconds, for an entry in the changeset.
- // `id` is guaranteed to be in the set.
- getModifiedTimestamp(id) {
- return this.changes[id];
- }
-
- // Adds a change for a tracked ID to the changeset.
- set(id, change) {
- this.changes[id] = change;
- }
-
- // Indicates whether an entry is in the changeset.
- has(id) {
- return id in this.changes;
}
-
- // Deletes an entry from the changeset. Used to clean up entries for
- // reconciled and successfully uploaded records.
- delete(id) {
- delete this.changes[id];
- }
-
- // Swaps two entries in the changeset. Used when reconciling duplicates that
- // have local changes.
- swap(oldID, newID) {
- this.changes[newID] = this.changes[oldID];
- delete this.changes[oldID];
- }
-
- // Returns an array of all tracked IDs in this changeset.
- ids() {
- return Object.keys(this.changes);
- }
-
- // Returns an array of `[id, change]` tuples. Used to repopulate the tracker
- // with entries for failed uploads at the end of a sync.
- entries() {
- return Object.entries(this.changes);
- }
-
- // Returns the number of entries in this changeset.
- count() {
- return this.ids().length;
- }
-
- // Clears the changeset.
- clear() {
- this.changes = {};
- }
-}
+};
diff --git a/services/sync/modules/engines/addons.js b/services/sync/modules/engines/addons.js
index 01dab58d1..ab3131c30 100644
--- a/services/sync/modules/engines/addons.js
+++ b/services/sync/modules/engines/addons.js
@@ -25,18 +25,15 @@
*
* Synchronization is influenced by the following preferences:
*
+ * - services.sync.addons.ignoreRepositoryChecking
* - services.sync.addons.ignoreUserEnabledChanges
* - services.sync.addons.trustedSourceHostnames
*
- * and also influenced by whether addons have repository caching enabled and
- * whether they allow installation of addons from insecure options (both of
- * which are themselves influenced by the "extensions." pref branch)
- *
* See the documentation in services-sync.js for the behavior of these prefs.
*/
"use strict";
-var {classes: Cc, interfaces: Ci, utils: Cu} = Components;
+const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
Cu.import("resource://services-sync/addonutils.js");
Cu.import("resource://services-sync/addonsreconciler.js");
@@ -44,7 +41,6 @@ Cu.import("resource://services-sync/engines.js");
Cu.import("resource://services-sync/record.js");
Cu.import("resource://services-sync/util.js");
Cu.import("resource://services-sync/constants.js");
-Cu.import("resource://services-sync/collection_validator.js");
Cu.import("resource://services-common/async.js");
Cu.import("resource://gre/modules/Preferences.jsm");
@@ -54,7 +50,7 @@ XPCOMUtils.defineLazyModuleGetter(this, "AddonManager",
XPCOMUtils.defineLazyModuleGetter(this, "AddonRepository",
"resource://gre/modules/addons/AddonRepository.jsm");
-this.EXPORTED_SYMBOLS = ["AddonsEngine", "AddonValidator"];
+this.EXPORTED_SYMBOLS = ["AddonsEngine"];
// 7 days in milliseconds.
const PRUNE_ADDON_CHANGES_THRESHOLD = 60 * 60 * 24 * 7 * 1000;
@@ -154,7 +150,7 @@ AddonsEngine.prototype = {
*/
getChangedIDs: function getChangedIDs() {
let changes = {};
- for (let [id, modified] of Object.entries(this._tracker.changedIDs)) {
+ for (let [id, modified] in Iterator(this._tracker.changedIDs)) {
changes[id] = modified;
}
@@ -164,7 +160,7 @@ AddonsEngine.prototype = {
// we assume this function is only called from within a sync.
let reconcilerChanges = this._reconciler.getChangesSinceDate(lastSyncDate);
let addons = this._reconciler.addons;
- for (let change of reconcilerChanges) {
+ for each (let change in reconcilerChanges) {
let changeTime = change[0];
let id = change[2];
@@ -177,7 +173,7 @@ AddonsEngine.prototype = {
continue;
}
- if (!this.isAddonSyncable(addons[id])) {
+ if (!this._store.isAddonSyncable(addons[id])) {
continue;
}
@@ -235,10 +231,6 @@ AddonsEngine.prototype = {
let cb = Async.makeSpinningCallback();
this._reconciler.refreshGlobalState(cb);
cb.wait();
- },
-
- isAddonSyncable(addon, ignoreRepoCheck) {
- return this._store.isAddonSyncable(addon, ignoreRepoCheck);
}
};
@@ -286,14 +278,6 @@ AddonsStore.prototype = {
}
}
- // Ignore incoming records for which an existing non-syncable addon
- // exists.
- let existingMeta = this.reconciler.addons[record.addonID];
- if (existingMeta && !this.isAddonSyncable(existingMeta)) {
- this._log.info("Ignoring incoming record for an existing but non-syncable addon", record.addonID);
- return;
- }
-
Store.prototype.applyIncoming.call(this, record);
},
@@ -307,23 +291,15 @@ AddonsStore.prototype = {
id: record.addonID,
syncGUID: record.id,
enabled: record.enabled,
- requireSecureURI: this._extensionsPrefs.get("install.requireSecureOrigin", true),
+ requireSecureURI: !Svc.Prefs.get("addons.ignoreRepositoryChecking", false),
}], cb);
// This will throw if there was an error. This will get caught by the sync
// engine and the record will try to be applied later.
let results = cb.wait();
- if (results.skipped.includes(record.addonID)) {
- this._log.info("Add-on skipped: " + record.addonID);
- // Just early-return for skipped addons - we don't want to arrange to
- // try again next time because the condition that caused up to skip
- // will remain true for this addon forever.
- return;
- }
-
let addon;
- for (let a of results.addons) {
+ for each (let a in results.addons) {
if (a.id == record.addonID) {
addon = a;
break;
@@ -467,8 +443,7 @@ AddonsStore.prototype = {
let ids = {};
let addons = this.reconciler.addons;
- for (let id in addons) {
- let addon = addons[id];
+ for each (let addon in addons) {
if (this.isAddonSyncable(addon)) {
ids[addon.guid] = true;
}
@@ -499,7 +474,7 @@ AddonsStore.prototype = {
}
this._log.info("Uninstalling add-on as part of wipe: " + addon.id);
- Utils.catch.call(this, () => addon.uninstall())();
+ Utils.catch(addon.uninstall)();
}
},
@@ -538,22 +513,16 @@ AddonsStore.prototype = {
*
* @param addon
* Addon instance
- * @param ignoreRepoCheck
- * Should we skip checking the Addons repository (primarially useful
- * for testing and validation).
* @return Boolean indicating whether it is appropriate for Sync
*/
- isAddonSyncable: function isAddonSyncable(addon, ignoreRepoCheck = false) {
+ isAddonSyncable: function isAddonSyncable(addon) {
// Currently, we limit syncable add-ons to those that are:
// 1) In a well-defined set of types
// 2) Installed in the current profile
// 3) Not installed by a foreign entity (i.e. installed by the app)
// since they act like global extensions.
// 4) Is not a hotfix.
- // 5) The addons XPIProvider doesn't veto it (i.e not being installed in
- // the profile directory, or any other reasons it says the addon can't
- // be synced)
- // 6) Are installed from AMO
+ // 5) Are installed from AMO
// We could represent the test as a complex boolean expression. We go the
// verbose route so the failure reason is logged.
@@ -573,12 +542,6 @@ AddonsStore.prototype = {
return false;
}
- // If the addon manager says it's not syncable, we skip it.
- if (!addon.isSyncable) {
- this._log.debug(addon.id + " not syncable: vetoed by the addon manager.");
- return false;
- }
-
// This may be too aggressive. If an add-on is downloaded from AMO and
// manually placed in the profile directory, foreignInstall will be set.
// Arguably, that add-on should be syncable.
@@ -589,20 +552,15 @@ AddonsStore.prototype = {
}
// Ignore hotfix extensions (bug 741670). The pref may not be defined.
- // XXX - note that addon.isSyncable will be false for hotfix addons, so
- // this check isn't strictly necessary - except for Sync tests which aren't
- // setup to create a "real" hotfix addon. This can be removed once those
- // tests are fixed (but keeping it doesn't hurt either)
if (this._extensionsPrefs.get("hotfix.id", null) == addon.id) {
this._log.debug(addon.id + " not syncable: is a hotfix.");
return false;
}
- // If the AddonRepository's cache isn't enabled (which it typically isn't
- // in tests), getCachedAddonByID always returns null - so skip the check
- // in that case. We also provide a way to specifically opt-out of the check
- // even if the cache is enabled, which is used by the validators.
- if (ignoreRepoCheck || !AddonRepository.cacheEnabled) {
+ // We provide a back door to skip the repository checking of an add-on.
+ // This is utilized by the tests to make testing easier. Users could enable
+ // this, but it would sacrifice security.
+ if (Svc.Prefs.get("addons.ignoreRepositoryChecking", false)) {
return true;
}
@@ -745,69 +703,3 @@ AddonsTracker.prototype = {
this.reconciler.stopListening();
},
};
-
-class AddonValidator extends CollectionValidator {
- constructor(engine = null) {
- super("addons", "id", [
- "addonID",
- "enabled",
- "applicationID",
- "source"
- ]);
- this.engine = engine;
- }
-
- getClientItems() {
- return Promise.all([
- new Promise(resolve =>
- AddonManager.getAllAddons(resolve)),
- new Promise(resolve =>
- AddonManager.getAddonsWithOperationsByTypes(["extension", "theme"], resolve)),
- ]).then(([installed, addonsWithPendingOperation]) => {
- // Addons pending install won't be in the first list, but addons pending
- // uninstall/enable/disable will be in both lists.
- let all = new Map(installed.map(addon => [addon.id, addon]));
- for (let addon of addonsWithPendingOperation) {
- all.set(addon.id, addon);
- }
- // Convert to an array since Map.prototype.values returns an iterable
- return [...all.values()];
- });
- }
-
- normalizeClientItem(item) {
- let enabled = !item.userDisabled;
- if (item.pendingOperations & AddonManager.PENDING_ENABLE) {
- enabled = true;
- } else if (item.pendingOperations & AddonManager.PENDING_DISABLE) {
- enabled = false;
- }
- return {
- enabled,
- id: item.syncGUID,
- addonID: item.id,
- applicationID: Services.appinfo.ID,
- source: "amo", // check item.foreignInstall?
- original: item
- };
- }
-
- normalizeServerItem(item) {
- let guid = this.engine._findDupe(item);
- if (guid) {
- item.id = guid;
- }
- return item;
- }
-
- clientUnderstands(item) {
- return item.applicationID === Services.appinfo.ID;
- }
-
- syncedByClient(item) {
- return !item.original.hidden &&
- !item.original.isSystem &&
- !(item.original.pendingOperations & AddonManager.PENDING_UNINSTALL) &&
- this.engine.isAddonSyncable(item.original, true);
- }
-}
diff --git a/services/sync/modules/engines/bookmarks.js b/services/sync/modules/engines/bookmarks.js
index 76a198a8b..1936afc3f 100644
--- a/services/sync/modules/engines/bookmarks.js
+++ b/services/sync/modules/engines/bookmarks.js
@@ -6,12 +6,11 @@ this.EXPORTED_SYMBOLS = ['BookmarksEngine', "PlacesItem", "Bookmark",
"BookmarkFolder", "BookmarkQuery",
"Livemark", "BookmarkSeparator"];
-var Cc = Components.classes;
-var Ci = Components.interfaces;
-var Cu = Components.utils;
+const Cc = Components.classes;
+const Ci = Components.interfaces;
+const Cu = Components.utils;
Cu.import("resource://gre/modules/PlacesUtils.jsm");
-Cu.import("resource://gre/modules/PlacesSyncUtils.jsm");
Cu.import("resource://gre/modules/XPCOMUtils.jsm");
Cu.import("resource://services-common/async.js");
Cu.import("resource://services-sync/constants.js");
@@ -20,57 +19,21 @@ Cu.import("resource://services-sync/record.js");
Cu.import("resource://services-sync/util.js");
Cu.import("resource://gre/modules/Task.jsm");
Cu.import("resource://gre/modules/PlacesBackups.jsm");
-XPCOMUtils.defineLazyModuleGetter(this, "BookmarkValidator",
- "resource://services-sync/bookmark_validator.js");
-XPCOMUtils.defineLazyGetter(this, "PlacesBundle", () => {
- let bundleService = Cc["@mozilla.org/intl/stringbundle;1"]
- .getService(Ci.nsIStringBundleService);
- return bundleService.createBundle("chrome://places/locale/places.properties");
-});
-
-const ANNOS_TO_TRACK = [PlacesSyncUtils.bookmarks.DESCRIPTION_ANNO,
- PlacesSyncUtils.bookmarks.SIDEBAR_ANNO,
+
+const ALLBOOKMARKS_ANNO = "AllBookmarks";
+const DESCRIPTION_ANNO = "bookmarkProperties/description";
+const SIDEBAR_ANNO = "bookmarkProperties/loadInSidebar";
+const MOBILEROOT_ANNO = "mobile/bookmarksRoot";
+const MOBILE_ANNO = "MobileBookmarks";
+const EXCLUDEBACKUP_ANNO = "places/excludeFromBackup";
+const SMART_BOOKMARKS_ANNO = "Places/SmartBookmark";
+const PARENT_ANNO = "sync/parent";
+const ORGANIZERQUERY_ANNO = "PlacesOrganizer/OrganizerQuery";
+const ANNOS_TO_TRACK = [DESCRIPTION_ANNO, SIDEBAR_ANNO,
PlacesUtils.LMANNO_FEEDURI, PlacesUtils.LMANNO_SITEURI];
const SERVICE_NOT_SUPPORTED = "Service not supported on this platform";
const FOLDER_SORTINDEX = 1000000;
-const {
- SOURCE_SYNC,
- SOURCE_IMPORT,
- SOURCE_IMPORT_REPLACE,
-} = Ci.nsINavBookmarksService;
-
-const SQLITE_MAX_VARIABLE_NUMBER = 999;
-
-const ORGANIZERQUERY_ANNO = "PlacesOrganizer/OrganizerQuery";
-const ALLBOOKMARKS_ANNO = "AllBookmarks";
-const MOBILE_ANNO = "MobileBookmarks";
-
-// The tracker ignores changes made by bookmark import and restore, and
-// changes made by Sync. We don't need to exclude `SOURCE_IMPORT`, but both
-// import and restore fire `bookmarks-restore-*` observer notifications, and
-// the tracker doesn't currently distinguish between the two.
-const IGNORED_SOURCES = [SOURCE_SYNC, SOURCE_IMPORT, SOURCE_IMPORT_REPLACE];
-
-// Returns the constructor for a bookmark record type.
-function getTypeObject(type) {
- switch (type) {
- case "bookmark":
- case "microsummary":
- return Bookmark;
- case "query":
- return BookmarkQuery;
- case "folder":
- return BookmarkFolder;
- case "livemark":
- return Livemark;
- case "separator":
- return BookmarkSeparator;
- case "item":
- return PlacesItem;
- }
- return null;
-}
this.PlacesItem = function PlacesItem(collection, id, type) {
CryptoWrapper.call(this, collection, id);
@@ -89,32 +52,26 @@ PlacesItem.prototype = {
},
getTypeObject: function PlacesItem_getTypeObject(type) {
- let recordObj = getTypeObject(type);
- if (!recordObj) {
- throw new Error("Unknown places item object type: " + type);
+ switch (type) {
+ case "bookmark":
+ case "microsummary":
+ return Bookmark;
+ case "query":
+ return BookmarkQuery;
+ case "folder":
+ return BookmarkFolder;
+ case "livemark":
+ return Livemark;
+ case "separator":
+ return BookmarkSeparator;
+ case "item":
+ return PlacesItem;
}
- return recordObj;
+ throw "Unknown places item object type: " + type;
},
__proto__: CryptoWrapper.prototype,
_logName: "Sync.Record.PlacesItem",
-
- // Converts the record to a Sync bookmark object that can be passed to
- // `PlacesSyncUtils.bookmarks.{insert, update}`.
- toSyncBookmark() {
- return {
- kind: this.type,
- syncId: this.id,
- parentSyncId: this.parentid,
- };
- },
-
- // Populates the record from a Sync bookmark object returned from
- // `PlacesSyncUtils.bookmarks.fetch`.
- fromSyncBookmark(item) {
- this.parentid = item.parentSyncId;
- this.parentName = item.parentTitle;
- },
};
Utils.deferGetSet(PlacesItem,
@@ -127,27 +84,6 @@ this.Bookmark = function Bookmark(collection, id, type) {
Bookmark.prototype = {
__proto__: PlacesItem.prototype,
_logName: "Sync.Record.Bookmark",
-
- toSyncBookmark() {
- let info = PlacesItem.prototype.toSyncBookmark.call(this);
- info.title = this.title;
- info.url = this.bmkUri;
- info.description = this.description;
- info.loadInSidebar = this.loadInSidebar;
- info.tags = this.tags;
- info.keyword = this.keyword;
- return info;
- },
-
- fromSyncBookmark(item) {
- PlacesItem.prototype.fromSyncBookmark.call(this, item);
- this.title = item.title;
- this.bmkUri = item.url.href;
- this.description = item.description;
- this.loadInSidebar = item.loadInSidebar;
- this.tags = item.tags;
- this.keyword = item.keyword;
- },
};
Utils.deferGetSet(Bookmark,
@@ -161,19 +97,6 @@ this.BookmarkQuery = function BookmarkQuery(collection, id) {
BookmarkQuery.prototype = {
__proto__: Bookmark.prototype,
_logName: "Sync.Record.BookmarkQuery",
-
- toSyncBookmark() {
- let info = Bookmark.prototype.toSyncBookmark.call(this);
- info.folder = this.folderName;
- info.query = this.queryId;
- return info;
- },
-
- fromSyncBookmark(item) {
- Bookmark.prototype.fromSyncBookmark.call(this, item);
- this.folderName = item.folder;
- this.queryId = item.query;
- },
};
Utils.deferGetSet(BookmarkQuery,
@@ -186,20 +109,6 @@ this.BookmarkFolder = function BookmarkFolder(collection, id, type) {
BookmarkFolder.prototype = {
__proto__: PlacesItem.prototype,
_logName: "Sync.Record.Folder",
-
- toSyncBookmark() {
- let info = PlacesItem.prototype.toSyncBookmark.call(this);
- info.description = this.description;
- info.title = this.title;
- return info;
- },
-
- fromSyncBookmark(item) {
- PlacesItem.prototype.fromSyncBookmark.call(this, item);
- this.title = item.title;
- this.description = item.description;
- this.children = item.childSyncIds;
- },
};
Utils.deferGetSet(BookmarkFolder, "cleartext", ["description", "title",
@@ -211,21 +120,6 @@ this.Livemark = function Livemark(collection, id) {
Livemark.prototype = {
__proto__: BookmarkFolder.prototype,
_logName: "Sync.Record.Livemark",
-
- toSyncBookmark() {
- let info = BookmarkFolder.prototype.toSyncBookmark.call(this);
- info.feed = this.feedUri;
- info.site = this.siteUri;
- return info;
- },
-
- fromSyncBookmark(item) {
- BookmarkFolder.prototype.fromSyncBookmark.call(this, item);
- this.feedUri = item.feed.href;
- if (item.site) {
- this.siteUri = item.site.href;
- }
- },
};
Utils.deferGetSet(Livemark, "cleartext", ["siteUri", "feedUri"]);
@@ -236,15 +130,81 @@ this.BookmarkSeparator = function BookmarkSeparator(collection, id) {
BookmarkSeparator.prototype = {
__proto__: PlacesItem.prototype,
_logName: "Sync.Record.Separator",
-
- fromSyncBookmark(item) {
- PlacesItem.prototype.fromSyncBookmark.call(this, item);
- this.pos = item.index;
- },
};
Utils.deferGetSet(BookmarkSeparator, "cleartext", "pos");
+
+let kSpecialIds = {
+
+ // Special IDs. Note that mobile can attempt to create a record on
+ // dereference; special accessors are provided to prevent recursion within
+ // observers.
+ guids: ["menu", "places", "tags", "toolbar", "unfiled", "mobile"],
+
+ // Create the special mobile folder to store mobile bookmarks.
+ createMobileRoot: function createMobileRoot() {
+ let root = PlacesUtils.placesRootId;
+ let mRoot = PlacesUtils.bookmarks.createFolder(root, "mobile", -1);
+ PlacesUtils.annotations.setItemAnnotation(
+ mRoot, MOBILEROOT_ANNO, 1, 0, PlacesUtils.annotations.EXPIRE_NEVER);
+ PlacesUtils.annotations.setItemAnnotation(
+ mRoot, EXCLUDEBACKUP_ANNO, 1, 0, PlacesUtils.annotations.EXPIRE_NEVER);
+ return mRoot;
+ },
+
+ findMobileRoot: function findMobileRoot(create) {
+ // Use the (one) mobile root if it already exists.
+ let root = PlacesUtils.annotations.getItemsWithAnnotation(MOBILEROOT_ANNO, {});
+ if (root.length != 0)
+ return root[0];
+
+ if (create)
+ return this.createMobileRoot();
+
+ return null;
+ },
+
+ // Accessors for IDs.
+ isSpecialGUID: function isSpecialGUID(g) {
+ return this.guids.indexOf(g) != -1;
+ },
+
+ specialIdForGUID: function specialIdForGUID(guid, create) {
+ if (guid == "mobile") {
+ return this.findMobileRoot(create);
+ }
+ return this[guid];
+ },
+
+ // Don't bother creating mobile: if it doesn't exist, this ID can't be it!
+ specialGUIDForId: function specialGUIDForId(id) {
+ for each (let guid in this.guids)
+ if (this.specialIdForGUID(guid, false) == id)
+ return guid;
+ return null;
+ },
+
+ get menu() {
+ return PlacesUtils.bookmarksMenuFolderId;
+ },
+ get places() {
+ return PlacesUtils.placesRootId;
+ },
+ get tags() {
+ return PlacesUtils.tagsFolderId;
+ },
+ get toolbar() {
+ return PlacesUtils.toolbarFolderId;
+ },
+ get unfiled() {
+ return PlacesUtils.unfiledBookmarksFolderId;
+ },
+ get mobile() {
+ return this.findMobileRoot(true);
+ },
+};
+
this.BookmarksEngine = function BookmarksEngine(service) {
SyncEngine.call(this, "Bookmarks", service);
}
@@ -257,103 +217,68 @@ BookmarksEngine.prototype = {
_defaultSort: "index",
syncPriority: 4,
- allowSkippedRecord: false,
-
- // A diagnostic helper to get the string value for a bookmark's URL given
- // its ID. Always returns a string - on error will return a string in the
- // form of "<description of error>" as this is purely for, eg, logging.
- // (This means hitting the DB directly and we don't bother using a cached
- // statement - we should rarely hit this.)
- _getStringUrlForId(id) {
- let url;
- try {
- let stmt = this._store._getStmt(`
- SELECT h.url
- FROM moz_places h
- JOIN moz_bookmarks b ON h.id = b.fk
- WHERE b.id = :id`);
- stmt.params.id = id;
- let rows = Async.querySpinningly(stmt, ["url"]);
- url = rows.length == 0 ? "<not found>" : rows[0].url;
- } catch (ex) {
- if (Async.isShutdownException(ex)) {
- throw ex;
- }
- if (ex instanceof Ci.mozIStorageError) {
- url = `<failed: Storage error: ${ex.message} (${ex.result})>`;
- } else {
- url = `<failed: ${ex.toString()}>`;
- }
- }
- return url;
- },
- _guidMapFailed: false,
- _buildGUIDMap: function _buildGUIDMap() {
- let store = this._store;
- let guidMap = {};
- let tree = Async.promiseSpinningly(PlacesUtils.promiseBookmarksTree("", {
- includeItemIds: true
- }));
- function* walkBookmarksTree(tree, parent=null) {
- if (tree) {
- // Skip root node
- if (parent) {
- yield [tree, parent];
+ _sync: function _sync() {
+ let engine = this;
+ let batchEx = null;
+
+ // Try running sync in batch mode
+ PlacesUtils.bookmarks.runInBatchMode({
+ runBatched: function wrappedSync() {
+ try {
+ SyncEngine.prototype._sync.call(engine);
}
- if (tree.children) {
- for (let child of tree.children) {
- store._sleep(0); // avoid jank while looping.
- yield* walkBookmarksTree(child, tree);
- }
+ catch(ex) {
+ batchEx = ex;
}
}
- }
+ }, null);
- function* walkBookmarksRoots(tree, rootIDs) {
- for (let id of rootIDs) {
- let bookmarkRoot = tree.children.find(child => child.id === id);
- if (bookmarkRoot === null) {
- continue;
- }
- yield* walkBookmarksTree(bookmarkRoot, tree);
- }
+ // Expose the exception if something inside the batch failed
+ if (batchEx != null) {
+ throw batchEx;
}
+ },
- let rootsToWalk = getChangeRootIds();
-
- for (let [node, parent] of walkBookmarksRoots(tree, rootsToWalk)) {
- let {guid, id, type: placeType} = node;
- guid = PlacesSyncUtils.bookmarks.guidToSyncId(guid);
+ _guidMapFailed: false,
+ _buildGUIDMap: function _buildGUIDMap() {
+ let guidMap = {};
+ for (let guid in this._store.getAllIDs()) {
+ // Figure out with which key to store the mapping.
let key;
- switch (placeType) {
- case PlacesUtils.TYPE_X_MOZ_PLACE:
- // Bookmark
- let query = null;
- if (node.annos && node.uri.startsWith("place:")) {
- query = node.annos.find(({name}) =>
- name === PlacesSyncUtils.bookmarks.SMART_BOOKMARKS_ANNO);
- }
- if (query && query.value) {
- key = "q" + query.value;
- } else {
- key = "b" + node.uri + ":" + (node.title || "");
- }
+ let id = this._store.idForGUID(guid);
+ switch (PlacesUtils.bookmarks.getItemType(id)) {
+ case PlacesUtils.bookmarks.TYPE_BOOKMARK:
+
+ // Smart bookmarks map to their annotation value.
+ let queryId;
+ try {
+ queryId = PlacesUtils.annotations.getItemAnnotation(
+ id, SMART_BOOKMARKS_ANNO);
+ } catch(ex) {}
+
+ if (queryId)
+ key = "q" + queryId;
+ else
+ key = "b" + PlacesUtils.bookmarks.getBookmarkURI(id).spec + ":" +
+ PlacesUtils.bookmarks.getItemTitle(id);
break;
- case PlacesUtils.TYPE_X_MOZ_PLACE_CONTAINER:
- // Folder
- key = "f" + (node.title || "");
+ case PlacesUtils.bookmarks.TYPE_FOLDER:
+ key = "f" + PlacesUtils.bookmarks.getItemTitle(id);
break;
- case PlacesUtils.TYPE_X_MOZ_PLACE_SEPARATOR:
- // Separator
- key = "s" + node.index;
+ case PlacesUtils.bookmarks.TYPE_SEPARATOR:
+ key = "s" + PlacesUtils.bookmarks.getItemIndex(id);
break;
default:
- this._log.error("Unknown place type: '"+placeType+"'");
continue;
}
- let parentName = parent.title || "";
+ // The mapping is on a per parent-folder-name basis.
+ let parent = PlacesUtils.bookmarks.getFolderIdForItem(id);
+ if (parent <= 0)
+ continue;
+
+ let parentName = PlacesUtils.bookmarks.getItemTitle(parent);
if (guidMap[parentName] == null)
guidMap[parentName] = {};
@@ -381,17 +306,17 @@ BookmarksEngine.prototype = {
// hack should get them to dupe correctly.
if (item.queryId) {
key = "q" + item.queryId;
- altKey = "b" + item.bmkUri + ":" + (item.title || "");
+ altKey = "b" + item.bmkUri + ":" + item.title;
break;
}
// No queryID? Fall through to the regular bookmark case.
case "bookmark":
case "microsummary":
- key = "b" + item.bmkUri + ":" + (item.title || "");
+ key = "b" + item.bmkUri + ":" + item.title;
break;
case "folder":
case "livemark":
- key = "f" + (item.title || "");
+ key = "f" + item.title;
break;
case "separator":
key = "s" + item.pos;
@@ -405,22 +330,21 @@ BookmarksEngine.prototype = {
let guidMap = this._guidMap;
// Give the GUID if we have the matching pair.
- let parentName = item.parentName || "";
- this._log.trace("Finding mapping: " + parentName + ", " + key);
- let parent = guidMap[parentName];
-
+ this._log.trace("Finding mapping: " + item.parentName + ", " + key);
+ let parent = guidMap[item.parentName];
+
if (!parent) {
this._log.trace("No parent => no dupe.");
return undefined;
}
-
+
let dupe = parent[key];
-
+
if (dupe) {
this._log.trace("Mapped dupe: " + dupe);
return dupe;
}
-
+
if (altKey) {
dupe = parent[altKey];
if (dupe) {
@@ -428,7 +352,7 @@ BookmarksEngine.prototype = {
return dupe;
}
}
-
+
this._log.trace("No dupe found for key " + key + "/" + altKey + ".");
return undefined;
},
@@ -437,7 +361,7 @@ BookmarksEngine.prototype = {
SyncEngine.prototype._syncStartup.call(this);
let cb = Async.makeSpinningCallback();
- Task.spawn(function* () {
+ Task.spawn(function() {
// For first-syncs, make a backup for the user to restore
if (this.lastSync == 0) {
this._log.debug("Bookmarks backup starting.");
@@ -449,7 +373,8 @@ BookmarksEngine.prototype = {
// Failure to create a backup is somewhat bad, but probably not bad
// enough to prevent syncing of bookmarks - so just log the error and
// continue.
- this._log.warn("Error while backing up bookmarks, but continuing with sync", ex);
+ this._log.warn("Got exception \"" + Utils.exceptionStr(ex) +
+ "\" backing up bookmarks, but continuing with sync.");
cb();
}
);
@@ -464,10 +389,9 @@ BookmarksEngine.prototype = {
try {
guidMap = this._buildGUIDMap();
} catch (ex) {
- if (Async.isShutdownException(ex)) {
- throw ex;
- }
- this._log.warn("Error while building GUID map, skipping all other incoming items", ex);
+ this._log.warn("Got exception \"" + Utils.exceptionStr(ex) +
+ "\" building GUID map." +
+ " Skipping all other incoming items.");
throw {code: Engine.prototype.eEngineAbortApplyIncoming,
cause: ex};
}
@@ -476,71 +400,17 @@ BookmarksEngine.prototype = {
});
this._store._childrenToOrder = {};
- this._store.clearPendingDeletions();
- },
-
- _deletePending() {
- // Delete pending items -- See the comment above BookmarkStore's deletePending
- let newlyModified = Async.promiseSpinningly(this._store.deletePending());
- let now = this._tracker._now();
- this._log.debug("Deleted pending items", newlyModified);
- for (let modifiedSyncID of newlyModified) {
- if (!this._modified.has(modifiedSyncID)) {
- this._modified.set(modifiedSyncID, { timestamp: now, deleted: false });
- }
- }
- },
-
- // We avoid reviving folders since reviving them properly would require
- // reviving their children as well. Unfortunately, this is the wrong choice
- // in the case of a bookmark restore where wipeServer failed -- if the
- // server has the folder as deleted, we *would* want to reupload this folder.
- // This is mitigated by the fact that we move any undeleted children to the
- // grandparent when deleting the parent.
- _shouldReviveRemotelyDeletedRecord(item) {
- let kind = Async.promiseSpinningly(
- PlacesSyncUtils.bookmarks.getKindForSyncId(item.id));
- if (kind === PlacesSyncUtils.bookmarks.KINDS.FOLDER) {
- return false;
- }
-
- // In addition to preventing the deletion of this record (handled by the caller),
- // we need to mark the parent of this record for uploading next sync, in order
- // to ensure its children array is accurate.
- let modifiedTimestamp = this._modified.getModifiedTimestamp(item.id);
- if (!modifiedTimestamp) {
- // We only expect this to be called with items locally modified, so
- // something strange is going on - play it safe and don't revive it.
- this._log.error("_shouldReviveRemotelyDeletedRecord called on unmodified item: " + item.id);
- return false;
- }
-
- let localID = this._store.idForGUID(item.id);
- let localParentID = PlacesUtils.bookmarks.getFolderIdForItem(localID);
- let localParentSyncID = this._store.GUIDForId(localParentID);
-
- this._log.trace(`Reviving item "${item.id}" and marking parent ${localParentSyncID} as modified.`);
-
- if (!this._modified.has(localParentSyncID)) {
- this._modified.set(localParentSyncID, {
- timestamp: modifiedTimestamp,
- deleted: false
- });
- }
- return true
},
_processIncoming: function (newitems) {
try {
SyncEngine.prototype._processIncoming.call(this, newitems);
} finally {
- try {
- this._deletePending();
- } finally {
- // Reorder children.
- this._store._orderChildren();
- delete this._store._childrenToOrder;
- }
+ // Reorder children.
+ this._tracker.ignoreAll = true;
+ this._store._orderChildren();
+ this._tracker.ignoreAll = false;
+ delete this._store._childrenToOrder;
}
},
@@ -575,154 +445,16 @@ BookmarksEngine.prototype = {
}
let mapped = this._mapDupe(item);
this._log.debug(item.id + " mapped to " + mapped);
- // We must return a string, not an object, and the entries in the GUIDMap
- // are created via "new String()" making them an object.
- return mapped ? mapped.toString() : mapped;
- },
-
- pullAllChanges() {
- return new BookmarksChangeset(this._store.getAllIDs());
- },
-
- pullNewChanges() {
- let modifiedGUIDs = this._getModifiedGUIDs();
- if (!modifiedGUIDs.length) {
- return new BookmarksChangeset(this._tracker.changedIDs);
- }
-
- // We don't use `PlacesUtils.promiseDBConnection` here because
- // `getChangedIDs` might be called while we're in a batch, meaning we
- // won't see any changes until the batch finishes and the transaction
- // commits.
- let db = PlacesUtils.history.QueryInterface(Ci.nsPIPlacesDatabase)
- .DBConnection;
-
- // Filter out tags, organizer queries, and other descendants that we're
- // not tracking. We chunk `modifiedGUIDs` because SQLite limits the number
- // of bound parameters per query.
- for (let startIndex = 0;
- startIndex < modifiedGUIDs.length;
- startIndex += SQLITE_MAX_VARIABLE_NUMBER) {
-
- let chunkLength = Math.min(SQLITE_MAX_VARIABLE_NUMBER,
- modifiedGUIDs.length - startIndex);
-
- let query = `
- WITH RECURSIVE
- modifiedGuids(guid) AS (
- VALUES ${new Array(chunkLength).fill("(?)").join(", ")}
- ),
- syncedItems(id) AS (
- VALUES ${getChangeRootIds().map(id => `(${id})`).join(", ")}
- UNION ALL
- SELECT b.id
- FROM moz_bookmarks b
- JOIN syncedItems s ON b.parent = s.id
- )
- SELECT b.guid
- FROM modifiedGuids m
- JOIN moz_bookmarks b ON b.guid = m.guid
- LEFT JOIN syncedItems s ON b.id = s.id
- WHERE s.id IS NULL
- `;
-
- let statement = db.createAsyncStatement(query);
- try {
- for (let i = 0; i < chunkLength; i++) {
- statement.bindByIndex(i, modifiedGUIDs[startIndex + i]);
- }
- let results = Async.querySpinningly(statement, ["guid"]);
- for (let { guid } of results) {
- let syncID = PlacesSyncUtils.bookmarks.guidToSyncId(guid);
- this._tracker.removeChangedID(syncID);
- }
- } finally {
- statement.finalize();
- }
- }
-
- return new BookmarksChangeset(this._tracker.changedIDs);
- },
-
- // Returns an array of Places GUIDs for all changed items. Ignores deletions,
- // which won't exist in the DB and shouldn't be removed from the tracker.
- _getModifiedGUIDs() {
- let guids = [];
- for (let syncID in this._tracker.changedIDs) {
- if (this._tracker.changedIDs[syncID].deleted === true) {
- // The `===` check also filters out old persisted timestamps,
- // which won't have a `deleted` property.
- continue;
- }
- let guid = PlacesSyncUtils.bookmarks.syncIdToGuid(syncID);
- guids.push(guid);
- }
- return guids;
- },
-
- // Called when _findDupe returns a dupe item and the engine has decided to
- // switch the existing item to the new incoming item.
- _switchItemToDupe(localDupeGUID, incomingItem) {
- // We unconditionally change the item's ID in case the engine knows of
- // an item but doesn't expose it through itemExists. If the API
- // contract were stronger, this could be changed.
- this._log.debug("Switching local ID to incoming: " + localDupeGUID + " -> " +
- incomingItem.id);
- this._store.changeItemID(localDupeGUID, incomingItem.id);
-
- // And mark the parent as being modified. Given we de-dupe based on the
- // parent *name* it's possible the item having its GUID changed has a
- // different parent from the incoming record.
- // So we need to find the GUID of the local parent.
- let now = this._tracker._now();
- let localID = this._store.idForGUID(incomingItem.id);
- let localParentID = PlacesUtils.bookmarks.getFolderIdForItem(localID);
- let localParentGUID = this._store.GUIDForId(localParentID);
- this._modified.set(localParentGUID, { modified: now, deleted: false });
-
- // And we also add the parent as reflected in the incoming record as the
- // de-dupe process might have used an existing item in a different folder.
- // But only if the parent exists, otherwise we will upload a deleted item
- // when it might actually be valid, just unknown to us. Note that this
- // scenario will still leave us with inconsistent client and server states;
- // the incoming record on the server references a parent that isn't the
- // actual parent locally - see bug 1297955.
- if (localParentGUID != incomingItem.parentid) {
- let remoteParentID = this._store.idForGUID(incomingItem.parentid);
- if (remoteParentID > 0) {
- // The parent specified in the record does exist, so we are going to
- // attempt a move when we come to applying the record. Mark the parent
- // as being modified so we will later upload it with the new child
- // reference.
- this._modified.set(incomingItem.parentid, { modified: now, deleted: false });
- } else {
- // We aren't going to do a move as we don't have the parent (yet?).
- // When applying the record we will add our special PARENT_ANNO
- // annotation, so if it arrives in the future (either this Sync or a
- // later one) it will be reparented.
- this._log.debug(`Incoming duplicate item ${incomingItem.id} specifies ` +
- `non-existing parent ${incomingItem.parentid}`);
- }
- }
-
- // The local, duplicate ID is always deleted on the server - but for
- // bookmarks it is a logical delete.
- // Simply adding this (now non-existing) ID to the tracker is enough.
- this._modified.set(localDupeGUID, { modified: now, deleted: true });
- },
- getValidator() {
- return new BookmarkValidator();
+ return mapped;
}
};
function BookmarksStore(name, engine) {
Store.call(this, name, engine);
- this._foldersToDelete = new Set();
- this._atomsToDelete = new Set();
+
// Explicitly nullify our references to our cached services so we don't leak
Svc.Obs.add("places-shutdown", function() {
- for (let query in this._stmts) {
- let stmt = this._stmts[query];
+ for each (let [query, stmt] in Iterator(this._stmts)) {
stmt.finalize();
}
this._stmts = {};
@@ -732,12 +464,70 @@ BookmarksStore.prototype = {
__proto__: Store.prototype,
itemExists: function BStore_itemExists(id) {
- return this.idForGUID(id) > 0;
+ return this.idForGUID(id, true) > 0;
},
+
+ /*
+ * If the record is a tag query, rewrite it to refer to the local tag ID.
+ *
+ * Otherwise, just return.
+ */
+ preprocessTagQuery: function preprocessTagQuery(record) {
+ if (record.type != "query" ||
+ record.bmkUri == null ||
+ !record.folderName)
+ return;
+
+ // Yes, this works without chopping off the "place:" prefix.
+ let uri = record.bmkUri
+ let queriesRef = {};
+ let queryCountRef = {};
+ let optionsRef = {};
+ PlacesUtils.history.queryStringToQueries(uri, queriesRef, queryCountRef,
+ optionsRef);
+
+ // We only process tag URIs.
+ if (optionsRef.value.resultType != optionsRef.value.RESULTS_AS_TAG_CONTENTS)
+ return;
+
+ // Tag something to ensure that the tag exists.
+ let tag = record.folderName;
+ let dummyURI = Utils.makeURI("about:weave#BStore_preprocess");
+ PlacesUtils.tagging.tagURI(dummyURI, [tag]);
+
+ // Look for the id of the tag, which might just have been added.
+ let tags = this._getNode(PlacesUtils.tagsFolderId);
+ if (!(tags instanceof Ci.nsINavHistoryQueryResultNode)) {
+ this._log.debug("tags isn't an nsINavHistoryQueryResultNode; aborting.");
+ return;
+ }
+ tags.containerOpen = true;
+ try {
+ for (let i = 0; i < tags.childCount; i++) {
+ let child = tags.getChild(i);
+ if (child.title == tag) {
+ // Found the tag, so fix up the query to use the right id.
+ this._log.debug("Tag query folder: " + tag + " = " + child.itemId);
+
+ this._log.trace("Replacing folders in: " + uri);
+ for each (let q in queriesRef.value)
+ q.setFolders([child.itemId], 1);
+
+ record.bmkUri = PlacesUtils.history.queriesToQueryString(
+ queriesRef.value, queryCountRef.value, optionsRef.value);
+ return;
+ }
+ }
+ }
+ finally {
+ tags.containerOpen = false;
+ }
+ },
+
applyIncoming: function BStore_applyIncoming(record) {
this._log.debug("Applying record " + record.id);
- let isSpecial = PlacesSyncUtils.bookmarks.ROOTS.includes(record.id);
+ let isSpecial = record.id in kSpecialIds;
if (record.deleted) {
if (isSpecial) {
@@ -765,217 +555,548 @@ BookmarksStore.prototype = {
return;
}
+ // Preprocess the record before doing the normal apply.
+ this.preprocessTagQuery(record);
+
// Figure out the local id of the parent GUID if available
let parentGUID = record.parentid;
if (!parentGUID) {
throw "Record " + record.id + " has invalid parentid: " + parentGUID;
}
- this._log.debug("Remote parent is " + parentGUID);
+ this._log.debug("Local parent is " + parentGUID);
+
+ let parentId = this.idForGUID(parentGUID);
+ if (parentId > 0) {
+ // Save the parent id for modifying the bookmark later
+ record._parent = parentId;
+ record._orphan = false;
+ this._log.debug("Record " + record.id + " is not an orphan.");
+ } else {
+ this._log.trace("Record " + record.id +
+ " is an orphan: could not find parent " + parentGUID);
+ record._orphan = true;
+ }
// Do the normal processing of incoming records
Store.prototype.applyIncoming.call(this, record);
- if (record.type == "folder" && record.children) {
- this._childrenToOrder[record.id] = record.children;
+ // Do some post-processing if we have an item
+ let itemId = this.idForGUID(record.id);
+ if (itemId > 0) {
+ // Move any children that are looking for this folder as a parent
+ if (record.type == "folder") {
+ this._reparentOrphans(itemId);
+ // Reorder children later
+ if (record.children)
+ this._childrenToOrder[record.id] = record.children;
+ }
+
+ // Create an annotation to remember that it needs reparenting.
+ if (record._orphan) {
+ PlacesUtils.annotations.setItemAnnotation(
+ itemId, PARENT_ANNO, parentGUID, 0,
+ PlacesUtils.annotations.EXPIRE_NEVER);
+ }
+ }
+ },
+
+ /**
+ * Find all ids of items that have a given value for an annotation
+ */
+ _findAnnoItems: function BStore__findAnnoItems(anno, val) {
+ return PlacesUtils.annotations.getItemsWithAnnotation(anno, {})
+ .filter(function(id) {
+ return PlacesUtils.annotations.getItemAnnotation(id, anno) == val;
+ });
+ },
+
+ /**
+ * For the provided parent item, attach its children to it
+ */
+ _reparentOrphans: function _reparentOrphans(parentId) {
+ // Find orphans and reunite with this folder parent
+ let parentGUID = this.GUIDForId(parentId);
+ let orphans = this._findAnnoItems(PARENT_ANNO, parentGUID);
+
+ this._log.debug("Reparenting orphans " + orphans + " to " + parentId);
+ orphans.forEach(function(orphan) {
+ // Move the orphan to the parent and drop the missing parent annotation
+ if (this._reparentItem(orphan, parentId)) {
+ PlacesUtils.annotations.removeItemAnnotation(orphan, PARENT_ANNO);
+ }
+ }, this);
+ },
+
+ _reparentItem: function _reparentItem(itemId, parentId) {
+ this._log.trace("Attempting to move item " + itemId + " to new parent " +
+ parentId);
+ try {
+ if (parentId > 0) {
+ PlacesUtils.bookmarks.moveItem(itemId, parentId,
+ PlacesUtils.bookmarks.DEFAULT_INDEX);
+ return true;
+ }
+ } catch(ex) {
+ this._log.debug("Failed to reparent item. " + Utils.exceptionStr(ex));
+ }
+ return false;
+ },
+
+ // Turn a record's nsINavBookmarksService constant and other attributes into
+ // a granular type for comparison.
+ _recordType: function _recordType(itemId) {
+ let bms = PlacesUtils.bookmarks;
+ let type = bms.getItemType(itemId);
+
+ switch (type) {
+ case bms.TYPE_FOLDER:
+ if (PlacesUtils.annotations
+ .itemHasAnnotation(itemId, PlacesUtils.LMANNO_FEEDURI)) {
+ return "livemark";
+ }
+ return "folder";
+
+ case bms.TYPE_BOOKMARK:
+ let bmkUri = bms.getBookmarkURI(itemId).spec;
+ if (bmkUri.indexOf("place:") == 0) {
+ return "query";
+ }
+ return "bookmark";
+
+ case bms.TYPE_SEPARATOR:
+ return "separator";
+
+ default:
+ return null;
}
},
create: function BStore_create(record) {
- let info = record.toSyncBookmark();
- // This can throw if we're inserting an invalid or incomplete bookmark.
- // That's fine; the exception will be caught by `applyIncomingBatch`
- // without aborting further processing.
- let item = Async.promiseSpinningly(PlacesSyncUtils.bookmarks.insert(info));
- if (item) {
- this._log.debug(`Created ${item.kind} ${item.syncId} under ${
- item.parentSyncId}`, item);
+ // Default to unfiled if we don't have the parent yet.
+
+ // Valid parent IDs are all positive integers. Other values -- undefined,
+ // null, -1 -- all compare false for > 0, so this catches them all. We
+ // don't just use <= without the !, because undefined and null compare
+ // false for that, too!
+ if (!(record._parent > 0)) {
+ this._log.debug("Parent is " + record._parent + "; reparenting to unfiled.");
+ record._parent = kSpecialIds.unfiled;
+ }
+
+ let newId;
+ switch (record.type) {
+ case "bookmark":
+ case "query":
+ case "microsummary": {
+ let uri = Utils.makeURI(record.bmkUri);
+ newId = PlacesUtils.bookmarks.insertBookmark(
+ record._parent, uri, PlacesUtils.bookmarks.DEFAULT_INDEX, record.title);
+ this._log.debug("created bookmark " + newId + " under " + record._parent
+ + " as " + record.title + " " + record.bmkUri);
+
+ // Smart bookmark annotations are strings.
+ if (record.queryId) {
+ PlacesUtils.annotations.setItemAnnotation(
+ newId, SMART_BOOKMARKS_ANNO, record.queryId, 0,
+ PlacesUtils.annotations.EXPIRE_NEVER);
+ }
+
+ if (Array.isArray(record.tags)) {
+ this._tagURI(uri, record.tags);
+ }
+ PlacesUtils.bookmarks.setKeywordForBookmark(newId, record.keyword);
+ if (record.description) {
+ PlacesUtils.annotations.setItemAnnotation(
+ newId, DESCRIPTION_ANNO, record.description, 0,
+ PlacesUtils.annotations.EXPIRE_NEVER);
+ }
+
+ if (record.loadInSidebar) {
+ PlacesUtils.annotations.setItemAnnotation(
+ newId, SIDEBAR_ANNO, true, 0,
+ PlacesUtils.annotations.EXPIRE_NEVER);
+ }
+
+ } break;
+ case "folder":
+ newId = PlacesUtils.bookmarks.createFolder(
+ record._parent, record.title, PlacesUtils.bookmarks.DEFAULT_INDEX);
+ this._log.debug("created folder " + newId + " under " + record._parent
+ + " as " + record.title);
+
+ if (record.description) {
+ PlacesUtils.annotations.setItemAnnotation(
+ newId, DESCRIPTION_ANNO, record.description, 0,
+ PlacesUtils.annotations.EXPIRE_NEVER);
+ }
+
+ // record.children will be dealt with in _orderChildren.
+ break;
+ case "livemark":
+ let siteURI = null;
+ if (!record.feedUri) {
+ this._log.debug("No feed URI: skipping livemark record " + record.id);
+ return;
+ }
+ if (PlacesUtils.annotations
+ .itemHasAnnotation(record._parent, PlacesUtils.LMANNO_FEEDURI)) {
+ this._log.debug("Invalid parent: skipping livemark record " + record.id);
+ return;
+ }
+
+ if (record.siteUri != null)
+ siteURI = Utils.makeURI(record.siteUri);
+
+ // Until this engine can handle asynchronous error reporting, we need to
+ // detect errors on creation synchronously.
+ let spinningCb = Async.makeSpinningCallback();
+
+ let livemarkObj = {title: record.title,
+ parentId: record._parent,
+ index: PlacesUtils.bookmarks.DEFAULT_INDEX,
+ feedURI: Utils.makeURI(record.feedUri),
+ siteURI: siteURI,
+ guid: record.id};
+ PlacesUtils.livemarks.addLivemark(livemarkObj).then(
+ aLivemark => { spinningCb(null, [Components.results.NS_OK, aLivemark]) },
+ () => { spinningCb(null, [Components.results.NS_ERROR_UNEXPECTED, aLivemark]) }
+ );
+
+ let [status, livemark] = spinningCb.wait();
+ if (!Components.isSuccessCode(status)) {
+ throw status;
+ }
+
+ this._log.debug("Created livemark " + livemark.id + " under " +
+ livemark.parentId + " as " + livemark.title +
+ ", " + livemark.siteURI.spec + ", " +
+ livemark.feedURI.spec + ", GUID " +
+ livemark.guid);
+ break;
+ case "separator":
+ newId = PlacesUtils.bookmarks.insertSeparator(
+ record._parent, PlacesUtils.bookmarks.DEFAULT_INDEX);
+ this._log.debug("created separator " + newId + " under " + record._parent);
+ break;
+ case "item":
+ this._log.debug(" -> got a generic places item.. do nothing?");
+ return;
+ default:
+ this._log.error("_create: Unknown item type: " + record.type);
+ return;
+ }
+
+ if (newId) {
+ // Livemarks can set the GUID through the API, so there's no need to
+ // do that here.
+ this._log.trace("Setting GUID of new item " + newId + " to " + record.id);
+ this._setGUID(newId, record.id);
+ }
+ },
+
+ // Factored out of `remove` to avoid redundant DB queries when the Places ID
+ // is already known.
+ removeById: function removeById(itemId, guid) {
+ let type = PlacesUtils.bookmarks.getItemType(itemId);
+
+ switch (type) {
+ case PlacesUtils.bookmarks.TYPE_BOOKMARK:
+ this._log.debug(" -> removing bookmark " + guid);
+ PlacesUtils.bookmarks.removeItem(itemId);
+ break;
+ case PlacesUtils.bookmarks.TYPE_FOLDER:
+ this._log.debug(" -> removing folder " + guid);
+ PlacesUtils.bookmarks.removeItem(itemId);
+ break;
+ case PlacesUtils.bookmarks.TYPE_SEPARATOR:
+ this._log.debug(" -> removing separator " + guid);
+ PlacesUtils.bookmarks.removeItem(itemId);
+ break;
+ default:
+ this._log.error("remove: Unknown item type: " + type);
+ break;
}
},
remove: function BStore_remove(record) {
- if (PlacesSyncUtils.bookmarks.isRootSyncID(record.id)) {
+ if (kSpecialIds.isSpecialGUID(record.id)) {
this._log.warn("Refusing to remove special folder " + record.id);
return;
}
- let recordKind = Async.promiseSpinningly(
- PlacesSyncUtils.bookmarks.getKindForSyncId(record.id));
- let isFolder = recordKind === PlacesSyncUtils.bookmarks.KINDS.FOLDER;
- this._log.trace(`Buffering removal of item "${record.id}" of type "${recordKind}".`);
- if (isFolder) {
- this._foldersToDelete.add(record.id);
- } else {
- this._atomsToDelete.add(record.id);
+
+ let itemId = this.idForGUID(record.id);
+ if (itemId <= 0) {
+ this._log.debug("Item " + record.id + " already removed");
+ return;
}
+ this.removeById(itemId, record.id);
},
- update: function BStore_update(record) {
- let info = record.toSyncBookmark();
- let item = Async.promiseSpinningly(PlacesSyncUtils.bookmarks.update(info));
- if (item) {
- this._log.debug(`Updated ${item.kind} ${item.syncId} under ${
- item.parentSyncId}`, item);
- }
+ _taggableTypes: ["bookmark", "microsummary", "query"],
+ isTaggable: function isTaggable(recordType) {
+ return this._taggableTypes.indexOf(recordType) != -1;
},
- _orderChildren: function _orderChildren() {
- let promises = Object.keys(this._childrenToOrder).map(syncID => {
- let children = this._childrenToOrder[syncID];
- return PlacesSyncUtils.bookmarks.order(syncID, children).catch(ex => {
- this._log.debug(`Could not order children for ${syncID}`, ex);
- });
- });
- Async.promiseSpinningly(Promise.all(promises));
- },
-
- // There's some complexity here around pending deletions. Our goals:
- //
- // - Don't delete any bookmarks a user has created but not explicitly deleted
- // (This includes any bookmark that was not a child of the folder at the
- // time the deletion was recorded, and also bookmarks restored from a backup).
- // - Don't undelete any bookmark without ensuring the server structure
- // includes it (see `BookmarkEngine.prototype._shouldReviveRemotelyDeletedRecord`)
- //
- // This leads the following approach:
- //
- // - Additions, moves, and updates are processed before deletions.
- // - To do this, all deletion operations are buffered during a sync. Folders
- // we plan on deleting have their sync id's stored in `this._foldersToDelete`,
- // and non-folders we plan on deleting have their sync id's stored in
- // `this._atomsToDelete`.
- // - The exception to this is the moves that occur to fix the order of bookmark
- // children, which are performed after we process deletions.
- // - Non-folders are deleted before folder deletions, so that when we process
- // folder deletions we know the correct state.
- // - Remote deletions always win for folders, but do not result in recursive
- // deletion of children. This is a hack because we're not able to distinguish
- // between value changes and structural changes to folders, and we don't even
- // have the old server record to compare to. See `BookmarkEngine`'s
- // `_shouldReviveRemotelyDeletedRecord` method.
- // - When a folder is deleted, its remaining children are moved in order to
- // their closest living ancestor. If this is interrupted (unlikely, but
- // possible given that we don't perform this operation in a transaction),
- // we revive the folder.
- // - Remote deletions can lose for non-folders, but only until we handle
- // bookmark restores correctly (removing stale state from the server -- this
- // is to say, if bug 1230011 is fixed, we should never revive bookmarks).
-
- deletePending: Task.async(function* deletePending() {
- yield this._deletePendingAtoms();
- let guidsToUpdate = yield this._deletePendingFolders();
- this.clearPendingDeletions();
- return guidsToUpdate;
- }),
-
- clearPendingDeletions() {
- this._foldersToDelete.clear();
- this._atomsToDelete.clear();
- },
-
- _deleteAtom: Task.async(function* _deleteAtom(syncID) {
- try {
- let info = yield PlacesSyncUtils.bookmarks.remove(syncID, {
- preventRemovalOfNonEmptyFolders: true
- });
- this._log.trace(`Removed item ${syncID} with type ${info.type}`);
- } catch (ex) {
- // Likely already removed.
- this._log.trace(`Error removing ${syncID}`, ex);
+ update: function BStore_update(record) {
+ let itemId = this.idForGUID(record.id);
+
+ if (itemId <= 0) {
+ this._log.debug("Skipping update for unknown item: " + record.id);
+ return;
}
- }),
-
- _deletePendingAtoms() {
- return Promise.all(
- [...this._atomsToDelete.values()]
- .map(syncID => this._deleteAtom(syncID)));
- },
-
- // Returns an array of sync ids that need updates.
- _deletePendingFolders: Task.async(function* _deletePendingFolders() {
- // To avoid data loss, we don't want to just delete the folder outright,
- // so we buffer folder deletions and process them at the end (now).
- //
- // At this point, any member in the folder that remains is either a folder
- // pending deletion (which we'll get to in this function), or an item that
- // should not be deleted. To avoid deleting these items, we first move them
- // to the parent of the folder we're about to delete.
- let needUpdate = new Set();
- for (let syncId of this._foldersToDelete) {
- let childSyncIds = yield PlacesSyncUtils.bookmarks.fetchChildSyncIds(syncId);
- if (!childSyncIds.length) {
- // No children -- just delete the folder.
- yield this._deleteAtom(syncId)
- continue;
- }
- // We could avoid some redundant work here by finding the nearest
- // grandparent who isn't present in `this._toDelete`...
- let grandparentSyncId = this.GUIDForId(
- PlacesUtils.bookmarks.getFolderIdForItem(
- this.idForGUID(PlacesSyncUtils.bookmarks.syncIdToGuid(syncId))));
+ // Two items are the same type if they have the same ItemType in Places,
+ // and also share some key characteristics (e.g., both being livemarks).
+ // We figure this out by examining the item to find the equivalent granular
+ // (string) type.
+ // If they're not the same type, we can't just update attributes. Delete
+ // then recreate the record instead.
+ let localItemType = this._recordType(itemId);
+ let remoteRecordType = record.type;
+ this._log.trace("Local type: " + localItemType + ". " +
+ "Remote type: " + remoteRecordType + ".");
+
+ if (localItemType != remoteRecordType) {
+ this._log.debug("Local record and remote record differ in type. " +
+ "Deleting and recreating.");
+ this.removeById(itemId, record.id);
+ this.create(record);
+ return;
+ }
- this._log.trace(`Moving ${childSyncIds.length} children of "${syncId}" to ` +
- `grandparent "${grandparentSyncId}" before deletion.`);
+ this._log.trace("Updating " + record.id + " (" + itemId + ")");
- // Move children out of the parent and into the grandparent
- yield Promise.all(childSyncIds.map(child => PlacesSyncUtils.bookmarks.update({
- syncId: child,
- parentSyncId: grandparentSyncId
- })));
+ // Move the bookmark to a new parent or new position if necessary
+ if (record._parent > 0 &&
+ PlacesUtils.bookmarks.getFolderIdForItem(itemId) != record._parent) {
+ this._reparentItem(itemId, record._parent);
+ }
- // Delete the (now empty) parent
- try {
- yield PlacesSyncUtils.bookmarks.remove(syncId, {
- preventRemovalOfNonEmptyFolders: true
- });
- } catch (e) {
- // We failed, probably because someone added something to this folder
- // between when we got the children and now (or the database is corrupt,
- // or something else happened...) This is unlikely, but possible. To
- // avoid corruption in this case, we need to reupload the record to the
- // server.
- //
- // (Ideally this whole operation would be done in a transaction, and this
- // wouldn't be possible).
- needUpdate.add(syncId);
+ for (let [key, val] in Iterator(record.cleartext)) {
+ switch (key) {
+ case "title":
+ PlacesUtils.bookmarks.setItemTitle(itemId, val);
+ break;
+ case "bmkUri":
+ PlacesUtils.bookmarks.changeBookmarkURI(itemId, Utils.makeURI(val));
+ break;
+ case "tags":
+ if (Array.isArray(val)) {
+ if (this.isTaggable(remoteRecordType)) {
+ this._tagID(itemId, val);
+ } else {
+ this._log.debug("Remote record type is invalid for tags: " + remoteRecordType);
+ }
+ }
+ break;
+ case "keyword":
+ PlacesUtils.bookmarks.setKeywordForBookmark(itemId, val);
+ break;
+ case "description":
+ if (val) {
+ PlacesUtils.annotations.setItemAnnotation(
+ itemId, DESCRIPTION_ANNO, val, 0,
+ PlacesUtils.annotations.EXPIRE_NEVER);
+ } else {
+ PlacesUtils.annotations.removeItemAnnotation(itemId, DESCRIPTION_ANNO);
+ }
+ break;
+ case "loadInSidebar":
+ if (val) {
+ PlacesUtils.annotations.setItemAnnotation(
+ itemId, SIDEBAR_ANNO, true, 0,
+ PlacesUtils.annotations.EXPIRE_NEVER);
+ } else {
+ PlacesUtils.annotations.removeItemAnnotation(itemId, SIDEBAR_ANNO);
+ }
+ break;
+ case "queryId":
+ PlacesUtils.annotations.setItemAnnotation(
+ itemId, SMART_BOOKMARKS_ANNO, val, 0,
+ PlacesUtils.annotations.EXPIRE_NEVER);
+ break;
}
+ }
+ },
- // Add children (for parentid) and grandparent (for children list) to set
- // of records needing an update, *unless* they're marked for deletion.
- if (!this._foldersToDelete.has(grandparentSyncId)) {
- needUpdate.add(grandparentSyncId);
- }
- for (let childSyncId of childSyncIds) {
- if (!this._foldersToDelete.has(childSyncId)) {
- needUpdate.add(childSyncId);
+ _orderChildren: function _orderChildren() {
+ for (let [guid, children] in Iterator(this._childrenToOrder)) {
+ // Reorder children according to the GUID list. Gracefully deal
+ // with missing items, e.g. locally deleted.
+ let delta = 0;
+ let parent = null;
+ for (let idx = 0; idx < children.length; idx++) {
+ let itemid = this.idForGUID(children[idx]);
+ if (itemid == -1) {
+ delta += 1;
+ this._log.trace("Could not locate record " + children[idx]);
+ continue;
+ }
+ try {
+ // This code path could be optimized by caching the parent earlier.
+ // Doing so should take in count any edge case due to reparenting
+ // or parent invalidations though.
+ if (!parent) {
+ parent = PlacesUtils.bookmarks.getFolderIdForItem(itemid);
+ }
+ PlacesUtils.bookmarks.moveItem(itemid, parent, idx - delta);
+ } catch (ex) {
+ this._log.debug("Could not move item " + children[idx] + ": " + ex);
}
}
}
- return [...needUpdate];
- }),
+ },
changeItemID: function BStore_changeItemID(oldID, newID) {
this._log.debug("Changing GUID " + oldID + " to " + newID);
- Async.promiseSpinningly(PlacesSyncUtils.bookmarks.changeGuid(oldID, newID));
+ // Make sure there's an item to change GUIDs
+ let itemId = this.idForGUID(oldID);
+ if (itemId <= 0)
+ return;
+
+ this._setGUID(itemId, newID);
+ },
+
+ _getNode: function BStore__getNode(folder) {
+ let query = PlacesUtils.history.getNewQuery();
+ query.setFolders([folder], 1);
+ return PlacesUtils.history.executeQuery(
+ query, PlacesUtils.history.getNewQueryOptions()).root;
+ },
+
+ _getTags: function BStore__getTags(uri) {
+ try {
+ if (typeof(uri) == "string")
+ uri = Utils.makeURI(uri);
+ } catch(e) {
+ this._log.warn("Could not parse URI \"" + uri + "\": " + e);
+ }
+ return PlacesUtils.tagging.getTagsForURI(uri, {});
+ },
+
+ _getDescription: function BStore__getDescription(id) {
+ try {
+ return PlacesUtils.annotations.getItemAnnotation(id, DESCRIPTION_ANNO);
+ } catch (e) {
+ return null;
+ }
+ },
+
+ _isLoadInSidebar: function BStore__isLoadInSidebar(id) {
+ return PlacesUtils.annotations.itemHasAnnotation(id, SIDEBAR_ANNO);
+ },
+
+ get _childGUIDsStm() {
+ return this._getStmt(
+ "SELECT id AS item_id, guid " +
+ "FROM moz_bookmarks " +
+ "WHERE parent = :parent " +
+ "ORDER BY position");
+ },
+ _childGUIDsCols: ["item_id", "guid"],
+
+ _getChildGUIDsForId: function _getChildGUIDsForId(itemid) {
+ let stmt = this._childGUIDsStm;
+ stmt.params.parent = itemid;
+ let rows = Async.querySpinningly(stmt, this._childGUIDsCols);
+ return rows.map(function (row) {
+ if (row.guid) {
+ return row.guid;
+ }
+ // A GUID hasn't been assigned to this item yet, do this now.
+ return this.GUIDForId(row.item_id);
+ }, this);
},
// Create a record starting from the weave id (places guid)
createRecord: function createRecord(id, collection) {
- let item = Async.promiseSpinningly(PlacesSyncUtils.bookmarks.fetch(id));
- if (!item) { // deleted item
- let record = new PlacesItem(collection, id);
+ let placeId = this.idForGUID(id);
+ let record;
+ if (placeId <= 0) { // deleted item
+ record = new PlacesItem(collection, id);
record.deleted = true;
return record;
}
- let recordObj = getTypeObject(item.kind);
- if (!recordObj) {
- this._log.warn("Unknown item type, cannot serialize: " + item.kind);
- recordObj = PlacesItem;
+ let parent = PlacesUtils.bookmarks.getFolderIdForItem(placeId);
+ switch (PlacesUtils.bookmarks.getItemType(placeId)) {
+ case PlacesUtils.bookmarks.TYPE_BOOKMARK:
+ let bmkUri = PlacesUtils.bookmarks.getBookmarkURI(placeId).spec;
+ if (bmkUri.indexOf("place:") == 0) {
+ record = new BookmarkQuery(collection, id);
+
+ // Get the actual tag name instead of the local itemId
+ let folder = bmkUri.match(/[:&]folder=(\d+)/);
+ try {
+ // There might not be the tag yet when creating on a new client
+ if (folder != null) {
+ folder = folder[1];
+ record.folderName = PlacesUtils.bookmarks.getItemTitle(folder);
+ this._log.trace("query id: " + folder + " = " + record.folderName);
+ }
+ }
+ catch(ex) {}
+
+ // Persist the Smart Bookmark anno, if found.
+ try {
+ let anno = PlacesUtils.annotations.getItemAnnotation(placeId, SMART_BOOKMARKS_ANNO);
+ if (anno != null) {
+ this._log.trace("query anno: " + SMART_BOOKMARKS_ANNO +
+ " = " + anno);
+ record.queryId = anno;
+ }
+ }
+ catch(ex) {}
+ }
+ else {
+ record = new Bookmark(collection, id);
+ }
+ record.title = PlacesUtils.bookmarks.getItemTitle(placeId);
+
+ record.parentName = PlacesUtils.bookmarks.getItemTitle(parent);
+ record.bmkUri = bmkUri;
+ record.tags = this._getTags(record.bmkUri);
+ record.keyword = PlacesUtils.bookmarks.getKeywordForBookmark(placeId);
+ record.description = this._getDescription(placeId);
+ record.loadInSidebar = this._isLoadInSidebar(placeId);
+ break;
+
+ case PlacesUtils.bookmarks.TYPE_FOLDER:
+ if (PlacesUtils.annotations
+ .itemHasAnnotation(placeId, PlacesUtils.LMANNO_FEEDURI)) {
+ record = new Livemark(collection, id);
+ let as = PlacesUtils.annotations;
+ record.feedUri = as.getItemAnnotation(placeId, PlacesUtils.LMANNO_FEEDURI);
+ try {
+ record.siteUri = as.getItemAnnotation(placeId, PlacesUtils.LMANNO_SITEURI);
+ } catch (ex) {}
+ } else {
+ record = new BookmarkFolder(collection, id);
+ }
+
+ if (parent > 0)
+ record.parentName = PlacesUtils.bookmarks.getItemTitle(parent);
+ record.title = PlacesUtils.bookmarks.getItemTitle(placeId);
+ record.description = this._getDescription(placeId);
+ record.children = this._getChildGUIDsForId(placeId);
+ break;
+
+ case PlacesUtils.bookmarks.TYPE_SEPARATOR:
+ record = new BookmarkSeparator(collection, id);
+ if (parent > 0)
+ record.parentName = PlacesUtils.bookmarks.getItemTitle(parent);
+ // Create a positioning identifier for the separator, used by _mapDupe
+ record.pos = PlacesUtils.bookmarks.getItemIndex(placeId);
+ break;
+
+ default:
+ record = new PlacesItem(collection, id);
+ this._log.warn("Unknown item type, cannot serialize: " +
+ PlacesUtils.bookmarks.getItemType(placeId));
}
- let record = new recordObj(collection, id);
- record.fromSyncBookmark(item);
+ record.parentid = this.GUIDForId(parent);
record.sortindex = this._calculateIndex(record);
return record;
@@ -997,22 +1118,84 @@ BookmarksStore.prototype = {
return this._getStmt(
"SELECT frecency " +
"FROM moz_places " +
- "WHERE url_hash = hash(:url) AND url = :url " +
+ "WHERE url = :url " +
"LIMIT 1");
},
_frecencyCols: ["frecency"],
+ get _setGUIDStm() {
+ return this._getStmt(
+ "UPDATE moz_bookmarks " +
+ "SET guid = :guid " +
+ "WHERE id = :item_id");
+ },
+
+ // Some helper functions to handle GUIDs
+ _setGUID: function _setGUID(id, guid) {
+ if (!guid)
+ guid = Utils.makeGUID();
+
+ let stmt = this._setGUIDStm;
+ stmt.params.guid = guid;
+ stmt.params.item_id = id;
+ Async.querySpinningly(stmt);
+ return guid;
+ },
+
+ get _guidForIdStm() {
+ return this._getStmt(
+ "SELECT guid " +
+ "FROM moz_bookmarks " +
+ "WHERE id = :item_id");
+ },
+ _guidForIdCols: ["guid"],
+
GUIDForId: function GUIDForId(id) {
- let guid = Async.promiseSpinningly(PlacesUtils.promiseItemGuid(id));
- return PlacesSyncUtils.bookmarks.guidToSyncId(guid);
+ let special = kSpecialIds.specialGUIDForId(id);
+ if (special)
+ return special;
+
+ let stmt = this._guidForIdStm;
+ stmt.params.item_id = id;
+
+ // Use the existing GUID if it exists
+ let result = Async.querySpinningly(stmt, this._guidForIdCols)[0];
+ if (result && result.guid)
+ return result.guid;
+
+ // Give the uri a GUID if it doesn't have one
+ return this._setGUID(id);
},
- idForGUID: function idForGUID(guid) {
- // guid might be a String object rather than a string.
- guid = PlacesSyncUtils.bookmarks.syncIdToGuid(guid.toString());
+ get _idForGUIDStm() {
+ return this._getStmt(
+ "SELECT id AS item_id " +
+ "FROM moz_bookmarks " +
+ "WHERE guid = :guid");
+ },
+ _idForGUIDCols: ["item_id"],
+
+ // noCreate is provided as an optional argument to prevent the creation of
+ // non-existent special records, such as "mobile".
+ idForGUID: function idForGUID(guid, noCreate) {
+ if (kSpecialIds.isSpecialGUID(guid))
+ return kSpecialIds.specialIdForGUID(guid, !noCreate);
- return Async.promiseSpinningly(PlacesUtils.promiseItemId(guid).catch(
- ex => -1));
+ let stmt = this._idForGUIDStm;
+ // guid might be a String object rather than a string.
+ stmt.params.guid = guid.toString();
+
+ let results = Async.querySpinningly(stmt, this._idForGUIDCols);
+ this._log.trace("Number of rows matching GUID " + guid + ": "
+ + results.length);
+
+ // Here's the one we care about: the first.
+ let result = results[0];
+
+ if (!result)
+ return -1;
+
+ return result.item_id;
},
_calculateIndex: function _calculateIndex(record) {
@@ -1037,48 +1220,107 @@ BookmarksStore.prototype = {
return index;
},
- getAllIDs: function BStore_getAllIDs() {
- let items = {};
-
- let query = `
- WITH RECURSIVE
- changeRootContents(id) AS (
- VALUES ${getChangeRootIds().map(id => `(${id})`).join(", ")}
- UNION ALL
- SELECT b.id
- FROM moz_bookmarks b
- JOIN changeRootContents c ON b.parent = c.id
- )
- SELECT guid
- FROM changeRootContents
- JOIN moz_bookmarks USING (id)
- `;
-
- let statement = this._getStmt(query);
- let results = Async.querySpinningly(statement, ["guid"]);
- for (let { guid } of results) {
- let syncID = PlacesSyncUtils.bookmarks.guidToSyncId(guid);
- items[syncID] = { modified: 0, deleted: false };
+ _getChildren: function BStore_getChildren(guid, items) {
+ let node = guid; // the recursion case
+ if (typeof(node) == "string") { // callers will give us the guid as the first arg
+ let nodeID = this.idForGUID(guid, true);
+ if (!nodeID) {
+ this._log.debug("No node for GUID " + guid + "; returning no children.");
+ return items;
+ }
+ node = this._getNode(nodeID);
+ }
+
+ if (node.type == node.RESULT_TYPE_FOLDER) {
+ node.QueryInterface(Ci.nsINavHistoryQueryResultNode);
+ node.containerOpen = true;
+ try {
+ // Remember all the children GUIDs and recursively get more
+ for (let i = 0; i < node.childCount; i++) {
+ let child = node.getChild(i);
+ items[this.GUIDForId(child.itemId)] = true;
+ this._getChildren(child, items);
+ }
+ }
+ finally {
+ node.containerOpen = false;
+ }
}
return items;
},
+ /**
+ * Associates the URI of the item with the provided ID with the
+ * provided array of tags.
+ * If the provided ID does not identify an item with a URI,
+ * returns immediately.
+ */
+ _tagID: function _tagID(itemID, tags) {
+ if (!itemID || !tags) {
+ return;
+ }
+
+ try {
+ let u = PlacesUtils.bookmarks.getBookmarkURI(itemID);
+ this._tagURI(u, tags);
+ } catch (e) {
+ this._log.warn("Got exception fetching URI for " + itemID + ": not tagging. " +
+ Utils.exceptionStr(e));
+
+ // I guess it doesn't have a URI. Don't try to tag it.
+ return;
+ }
+ },
+
+ /**
+ * Associate the provided URI with the provided array of tags.
+ * If the provided URI is falsy, returns immediately.
+ */
+ _tagURI: function _tagURI(bookmarkURI, tags) {
+ if (!bookmarkURI || !tags) {
+ return;
+ }
+
+ // Filter out any null/undefined/empty tags.
+ tags = tags.filter(t => t);
+
+ // Temporarily tag a dummy URI to preserve tag ids when untagging.
+ let dummyURI = Utils.makeURI("about:weave#BStore_tagURI");
+ PlacesUtils.tagging.tagURI(dummyURI, tags);
+ PlacesUtils.tagging.untagURI(bookmarkURI, null);
+ PlacesUtils.tagging.tagURI(bookmarkURI, tags);
+ PlacesUtils.tagging.untagURI(dummyURI, null);
+ },
+
+ getAllIDs: function BStore_getAllIDs() {
+ let items = {"menu": true,
+ "toolbar": true};
+ for each (let guid in kSpecialIds.guids) {
+ if (guid != "places" && guid != "tags")
+ this._getChildren(guid, items);
+ }
+ return items;
+ },
+
wipe: function BStore_wipe() {
- this.clearPendingDeletions();
- Async.promiseSpinningly(Task.spawn(function* () {
+ let cb = Async.makeSpinningCallback();
+ Task.spawn(function() {
// Save a backup before clearing out all bookmarks.
yield PlacesBackups.create(null, true);
- yield PlacesUtils.bookmarks.eraseEverything({
- source: SOURCE_SYNC,
- });
- }));
+ for each (let guid in kSpecialIds.guids)
+ if (guid != "places") {
+ let id = kSpecialIds.specialIdForGUID(guid);
+ if (id)
+ PlacesUtils.bookmarks.removeFolderChildren(id);
+ }
+ cb();
+ });
+ cb.wait();
}
};
function BookmarksTracker(name, engine) {
- this._batchDepth = 0;
- this._batchSawScoreIncrement = false;
Tracker.call(this, name, engine);
Svc.Obs.add("places-shutdown", this);
@@ -1086,16 +1328,6 @@ function BookmarksTracker(name, engine) {
BookmarksTracker.prototype = {
__proto__: Tracker.prototype,
- //`_ignore` checks the change source for each observer notification, so we
- // don't want to let the engine ignore all changes during a sync.
- get ignoreAll() {
- return false;
- },
-
- // Define an empty setter so that the engine doesn't throw a `TypeError`
- // setting a read-only property.
- set ignoreAll(value) {},
-
startTracking: function() {
PlacesUtils.bookmarks.addObserver(this, true);
Svc.Obs.add("bookmarks-restore-begin", this);
@@ -1116,9 +1348,11 @@ BookmarksTracker.prototype = {
switch (topic) {
case "bookmarks-restore-begin":
this._log.debug("Ignoring changes from importing bookmarks.");
+ this.ignoreAll = true;
break;
case "bookmarks-restore-success":
this._log.debug("Tracking all items on successful import.");
+ this.ignoreAll = false;
this._log.debug("Restore succeeded: wiping server and other clients.");
this.engine.service.resetClient([this.name]);
@@ -1127,6 +1361,7 @@ BookmarksTracker.prototype = {
break;
case "bookmarks-restore-failed":
this._log.debug("Tracking all items on failed import.");
+ this.ignoreAll = false;
break;
}
},
@@ -1137,68 +1372,73 @@ BookmarksTracker.prototype = {
Ci.nsISupportsWeakReference
]),
- addChangedID(id, change) {
- if (!id) {
- this._log.warn("Attempted to add undefined ID to tracker");
- return false;
- }
- if (this._ignored.includes(id)) {
- return false;
- }
- let shouldSaveChange = false;
- let currentChange = this.changedIDs[id];
- if (currentChange) {
- if (typeof currentChange == "number") {
- // Allow raw timestamps for backward-compatibility with persisted
- // changed IDs. The new format uses tuples to track deleted items.
- shouldSaveChange = currentChange < change.modified;
- } else {
- shouldSaveChange = currentChange.modified < change.modified ||
- currentChange.deleted != change.deleted;
- }
- } else {
- shouldSaveChange = true;
- }
- if (shouldSaveChange) {
- this._saveChangedID(id, change);
- }
- return true;
- },
-
/**
* Add a bookmark GUID to be uploaded and bump up the sync score.
*
- * @param itemId
- * The Places item ID of the bookmark to upload.
- * @param guid
- * The Places GUID of the bookmark to upload.
- * @param isTombstone
- * Whether we're uploading a tombstone for a removed bookmark.
+ * @param itemGuid
+ * GUID of the bookmark to upload.
*/
- _add: function BMT__add(itemId, guid, isTombstone = false) {
- let syncID = PlacesSyncUtils.bookmarks.guidToSyncId(guid);
- let info = { modified: Date.now() / 1000, deleted: isTombstone };
- if (this.addChangedID(syncID, info)) {
+ _add: function BMT__add(itemId, guid) {
+ guid = kSpecialIds.specialGUIDForId(itemId) || guid;
+ if (this.addChangedID(guid))
this._upScore();
- }
},
- /* Every add/remove/change will trigger a sync for MULTI_DEVICE (except in
- a batch operation, where we do it at the end of the batch) */
+ /* Every add/remove/change will trigger a sync for MULTI_DEVICE. */
_upScore: function BMT__upScore() {
- if (this._batchDepth == 0) {
- this.score += SCORE_INCREMENT_XLARGE;
- } else {
- this._batchSawScoreIncrement = true;
+ this.score += SCORE_INCREMENT_XLARGE;
+ },
+
+ /**
+ * Determine if a change should be ignored.
+ *
+ * @param itemId
+ * Item under consideration to ignore
+ * @param folder (optional)
+ * Folder of the item being changed
+ */
+ _ignore: function BMT__ignore(itemId, folder, guid) {
+ // Ignore unconditionally if the engine tells us to.
+ if (this.ignoreAll)
+ return true;
+
+ // Get the folder id if we weren't given one.
+ if (folder == null) {
+ try {
+ folder = PlacesUtils.bookmarks.getFolderIdForItem(itemId);
+ } catch (ex) {
+ this._log.debug("getFolderIdForItem(" + itemId +
+ ") threw; calling _ensureMobileQuery.");
+ // I'm guessing that gFIFI can throw, and perhaps that's why
+ // _ensureMobileQuery is here at all. Try not to call it.
+ this._ensureMobileQuery();
+ folder = PlacesUtils.bookmarks.getFolderIdForItem(itemId);
+ }
+ }
+
+ // Ignore changes to tags (folders under the tags folder).
+ let tags = kSpecialIds.tags;
+ if (folder == tags)
+ return true;
+
+ // Ignore tag items (the actual instance of a tag for a bookmark).
+ if (PlacesUtils.bookmarks.getFolderIdForItem(folder) == tags)
+ return true;
+
+ // Make sure to remove items that have the exclude annotation.
+ if (PlacesUtils.annotations.itemHasAnnotation(itemId, EXCLUDEBACKUP_ANNO)) {
+ this.removeChangedID(guid);
+ return true;
}
+
+ return false;
},
onItemAdded: function BMT_onItemAdded(itemId, folder, index,
itemType, uri, title, dateAdded,
- guid, parentGuid, source) {
- if (IGNORED_SOURCES.includes(source)) {
+ guid, parentGuid) {
+ if (this._ignore(itemId, folder, guid))
return;
- }
this._log.trace("onItemAdded: " + itemId);
this._add(itemId, guid);
@@ -1206,51 +1446,13 @@ BookmarksTracker.prototype = {
},
onItemRemoved: function (itemId, parentId, index, type, uri,
- guid, parentGuid, source) {
- if (IGNORED_SOURCES.includes(source)) {
- return;
- }
-
- // Ignore changes to tags (folders under the tags folder).
- if (parentId == PlacesUtils.tagsFolderId) {
- return;
- }
-
- let grandParentId = -1;
- try {
- grandParentId = PlacesUtils.bookmarks.getFolderIdForItem(parentId);
- } catch (ex) {
- // `getFolderIdForItem` can throw if the item no longer exists, such as
- // when we've removed a subtree using `removeFolderChildren`.
- return;
- }
-
- // Ignore tag items (the actual instance of a tag for a bookmark).
- if (grandParentId == PlacesUtils.tagsFolderId) {
+ guid, parentGuid) {
+ if (this._ignore(itemId, parentId, guid)) {
return;
}
- /**
- * The above checks are incomplete: we can still write tombstones for
- * items that we don't track, and upload extraneous roots.
- *
- * Consider the left pane root: it's a child of the Places root, and has
- * children and grandchildren. `PlacesUIUtils` can create, delete, and
- * recreate it as needed. We can't determine ancestors when the root or its
- * children are deleted, because they've already been removed from the
- * database when `onItemRemoved` is called. Likewise, we can't check their
- * "exclude from backup" annos, because they've *also* been removed.
- *
- * So, we end up writing tombstones for the left pane queries and left
- * pane root. For good measure, we'll also upload the Places root, because
- * it's the parent of the left pane root.
- *
- * As a workaround, we can track the parent GUID and reconstruct the item's
- * ancestry at sync time. This is complicated, and the previous behavior was
- * already wrong, so we'll wait for bug 1258127 to fix this generally.
- */
this._log.trace("onItemRemoved: " + itemId);
- this._add(itemId, guid, /* isTombstone */ true);
+ this._add(itemId, guid);
this._add(parentId, parentGuid);
},
@@ -1265,40 +1467,32 @@ BookmarksTracker.prototype = {
if (all.length == 0)
return;
+ // Disable handling of notifications while changing the mobile query
+ this.ignoreAll = true;
+
let mobile = find(MOBILE_ANNO);
- let queryURI = Utils.makeURI("place:folder=" + PlacesUtils.mobileFolderId);
- let title = PlacesBundle.GetStringFromName("MobileBookmarksFolderTitle");
+ let queryURI = Utils.makeURI("place:folder=" + kSpecialIds.mobile);
+ let title = Str.sync.get("mobile.label");
// Don't add OR remove the mobile bookmarks if there's nothing.
- if (PlacesUtils.bookmarks.getIdForItemAt(PlacesUtils.mobileFolderId, 0) == -1) {
+ if (PlacesUtils.bookmarks.getIdForItemAt(kSpecialIds.mobile, 0) == -1) {
if (mobile.length != 0)
- PlacesUtils.bookmarks.removeItem(mobile[0], SOURCE_SYNC);
+ PlacesUtils.bookmarks.removeItem(mobile[0]);
}
// Add the mobile bookmarks query if it doesn't exist
else if (mobile.length == 0) {
- let query = PlacesUtils.bookmarks.insertBookmark(all[0], queryURI, -1, title, /* guid */ null, SOURCE_SYNC);
+ let query = PlacesUtils.bookmarks.insertBookmark(all[0], queryURI, -1, title);
PlacesUtils.annotations.setItemAnnotation(query, ORGANIZERQUERY_ANNO, MOBILE_ANNO, 0,
- PlacesUtils.annotations.EXPIRE_NEVER, SOURCE_SYNC);
- PlacesUtils.annotations.setItemAnnotation(query, PlacesUtils.EXCLUDE_FROM_BACKUP_ANNO, 1, 0,
- PlacesUtils.annotations.EXPIRE_NEVER, SOURCE_SYNC);
+ PlacesUtils.annotations.EXPIRE_NEVER);
+ PlacesUtils.annotations.setItemAnnotation(query, EXCLUDEBACKUP_ANNO, 1, 0,
+ PlacesUtils.annotations.EXPIRE_NEVER);
}
- // Make sure the existing query URL and title are correct
- else {
- if (!PlacesUtils.bookmarks.getBookmarkURI(mobile[0]).equals(queryURI)) {
- PlacesUtils.bookmarks.changeBookmarkURI(mobile[0], queryURI,
- SOURCE_SYNC);
- }
- let queryTitle = PlacesUtils.bookmarks.getItemTitle(mobile[0]);
- if (queryTitle != title) {
- PlacesUtils.bookmarks.setItemTitle(mobile[0], title, SOURCE_SYNC);
- }
- let rootTitle =
- PlacesUtils.bookmarks.getItemTitle(PlacesUtils.mobileFolderId);
- if (rootTitle != title) {
- PlacesUtils.bookmarks.setItemTitle(PlacesUtils.mobileFolderId, title,
- SOURCE_SYNC);
- }
+ // Make sure the existing title is correct
+ else if (PlacesUtils.bookmarks.getItemTitle(mobile[0]) != title) {
+ PlacesUtils.bookmarks.setItemTitle(mobile[0], title);
}
+
+ this.ignoreAll = false;
},
// This method is oddly structured, but the idea is to return as quickly as
@@ -1306,11 +1500,10 @@ BookmarksTracker.prototype = {
// *each change*.
onItemChanged: function BMT_onItemChanged(itemId, property, isAnno, value,
lastModified, itemType, parentId,
- guid, parentGuid, oldValue,
- source) {
- if (IGNORED_SOURCES.includes(source)) {
+ guid, parentGuid) {
+ // Quicker checks first.
+ if (this.ignoreAll)
return;
- }
if (isAnno && (ANNOS_TO_TRACK.indexOf(property) == -1))
// Ignore annotations except for the ones that we sync.
@@ -1320,6 +1513,9 @@ BookmarksTracker.prototype = {
if (property == "favicon")
return;
+ if (this._ignore(itemId, parentId, guid))
+ return;
+
this._log.trace("onItemChanged: " + itemId +
(", " + property + (isAnno? " (anno)" : "")) +
(value ? (" = \"" + value + "\"") : ""));
@@ -1328,11 +1524,9 @@ BookmarksTracker.prototype = {
onItemMoved: function BMT_onItemMoved(itemId, oldParent, oldIndex,
newParent, newIndex, itemType,
- guid, oldParentGuid, newParentGuid,
- source) {
- if (IGNORED_SOURCES.includes(source)) {
+ guid, oldParentGuid, newParentGuid) {
+ if (this._ignore(itemId, newParent, guid))
return;
- }
this._log.trace("onItemMoved: " + itemId);
this._add(oldParent, oldParentGuid);
@@ -1342,37 +1536,10 @@ BookmarksTracker.prototype = {
}
// Remove any position annotations now that the user moved the item
- PlacesUtils.annotations.removeItemAnnotation(itemId,
- PlacesSyncUtils.bookmarks.SYNC_PARENT_ANNO, SOURCE_SYNC);
+ PlacesUtils.annotations.removeItemAnnotation(itemId, PARENT_ANNO);
},
- onBeginUpdateBatch: function () {
- ++this._batchDepth;
- },
- onEndUpdateBatch: function () {
- if (--this._batchDepth === 0 && this._batchSawScoreIncrement) {
- this.score += SCORE_INCREMENT_XLARGE;
- this._batchSawScoreIncrement = false;
- }
- },
+ onBeginUpdateBatch: function () {},
+ onEndUpdateBatch: function () {},
onItemVisited: function () {}
};
-
-// Returns an array of root IDs to recursively query for synced bookmarks.
-// Items in other roots, including tags and organizer queries, will be
-// ignored.
-function getChangeRootIds() {
- return [
- PlacesUtils.bookmarksMenuFolderId,
- PlacesUtils.toolbarFolderId,
- PlacesUtils.unfiledBookmarksFolderId,
- PlacesUtils.mobileFolderId,
- ];
-}
-
-class BookmarksChangeset extends Changeset {
- getModifiedTimestamp(id) {
- let change = this.changes[id];
- return change ? change.modified : Number.NaN;
- }
-}
diff --git a/services/sync/modules/engines/clients.js b/services/sync/modules/engines/clients.js
index 3dd679570..f423242c9 100644
--- a/services/sync/modules/engines/clients.js
+++ b/services/sync/modules/engines/clients.js
@@ -2,57 +2,24 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-/**
- * How does the clients engine work?
- *
- * - We use 2 files - commands.json and commands-syncing.json.
- *
- * - At sync upload time, we attempt a rename of commands.json to
- * commands-syncing.json, and ignore errors (helps for crash during sync!).
- * - We load commands-syncing.json and stash the contents in
- * _currentlySyncingCommands which lives for the duration of the upload process.
- * - We use _currentlySyncingCommands to build the outgoing records
- * - Immediately after successful upload, we delete commands-syncing.json from
- * disk (and clear _currentlySyncingCommands). We reconcile our local records
- * with what we just wrote in the server, and add failed IDs commands
- * back in commands.json
- * - Any time we need to "save" a command for future syncs, we load
- * commands.json, update it, and write it back out.
- */
-
this.EXPORTED_SYMBOLS = [
"ClientEngine",
"ClientsRec"
];
-var {classes: Cc, interfaces: Ci, utils: Cu} = Components;
+const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
-Cu.import("resource://services-common/async.js");
Cu.import("resource://services-common/stringbundle.js");
Cu.import("resource://services-sync/constants.js");
Cu.import("resource://services-sync/engines.js");
Cu.import("resource://services-sync/record.js");
-Cu.import("resource://services-sync/resource.js");
Cu.import("resource://services-sync/util.js");
-Cu.import("resource://gre/modules/Services.jsm");
-
-XPCOMUtils.defineLazyModuleGetter(this, "fxAccounts",
- "resource://gre/modules/FxAccounts.jsm");
const CLIENTS_TTL = 1814400; // 21 days
const CLIENTS_TTL_REFRESH = 604800; // 7 days
-const STALE_CLIENT_REMOTE_AGE = 604800; // 7 days
const SUPPORTED_PROTOCOL_VERSIONS = ["1.1", "1.5"];
-function hasDupeCommand(commands, action) {
- if (!commands) {
- return false;
- }
- return commands.some(other => other.command == action.command &&
- Utils.deepEquals(other.args, action.args));
-}
-
this.ClientsRec = function ClientsRec(collection, id) {
CryptoWrapper.call(this, collection, id);
}
@@ -66,27 +33,23 @@ Utils.deferGetSet(ClientsRec,
"cleartext",
["name", "type", "commands",
"version", "protocols",
- "formfactor", "os", "appPackage", "application", "device",
- "fxaDeviceId"]);
+ "formfactor", "os", "appPackage", "application", "device"]);
this.ClientEngine = function ClientEngine(service) {
SyncEngine.call(this, "Clients", service);
- // Reset the last sync timestamp on every startup so that we fetch all clients
- this.resetLastSync();
+ // Reset the client on every startup so that we fetch recent clients
+ this._resetClient();
}
ClientEngine.prototype = {
__proto__: SyncEngine.prototype,
_storeObj: ClientStore,
_recordObj: ClientsRec,
_trackerObj: ClientsTracker,
- allowSkippedRecord: false,
// Always sync client data as it controls other sync behavior
- get enabled() {
- return true;
- },
+ get enabled() true,
get lastRecordUpload() {
return Svc.Prefs.get(this.name + ".lastRecordUpload", 0);
@@ -95,31 +58,18 @@ ClientEngine.prototype = {
Svc.Prefs.set(this.name + ".lastRecordUpload", Math.floor(value));
},
- get remoteClients() {
- // return all non-stale clients for external consumption.
- return Object.values(this._store._remoteClients).filter(v => !v.stale);
- },
-
- remoteClientExists(id) {
- let client = this._store._remoteClients[id];
- return !!(client && !client.stale);
- },
-
// Aggregate some stats on the composition of clients on this account
get stats() {
let stats = {
- hasMobile: this.localType == DEVICE_TYPE_MOBILE,
+ hasMobile: this.localType == "mobile",
names: [this.localName],
numClients: 1,
};
- for (let id in this._store._remoteClients) {
- let {name, type, stale} = this._store._remoteClients[id];
- if (!stale) {
- stats.hasMobile = stats.hasMobile || type == DEVICE_TYPE_MOBILE;
- stats.names.push(name);
- stats.numClients++;
- }
+ for each (let {name, type} in this._store._remoteClients) {
+ stats.hasMobile = stats.hasMobile || type == "mobile";
+ stats.names.push(name);
+ stats.numClients++;
}
return stats;
@@ -135,11 +85,7 @@ ClientEngine.prototype = {
counts.set(this.localType, 1);
- for (let id in this._store._remoteClients) {
- let record = this._store._remoteClients[id];
- if (record.stale) {
- continue; // pretend "stale" records don't exist.
- }
+ for each (let record in this._store._remoteClients) {
let type = record.type;
if (!counts.has(type)) {
counts.set(type, 0);
@@ -156,9 +102,7 @@ ClientEngine.prototype = {
let localID = Svc.Prefs.get("client.GUID", "");
return localID == "" ? this.localID = Utils.makeGUID() : localID;
},
- set localID(value) {
- Svc.Prefs.set("client.GUID", value);
- },
+ set localID(value) Svc.Prefs.set("client.GUID", value),
get brandName() {
let brand = new StringBundle("chrome://branding/locale/brand.properties");
@@ -166,97 +110,23 @@ ClientEngine.prototype = {
},
get localName() {
- let name = Utils.getDeviceName();
- // If `getDeviceName` returns the default name, set the pref. FxA registers
- // the device before syncing, so we don't need to update the registration
- // in this case.
- Svc.Prefs.set("client.name", name);
- return name;
- },
- set localName(value) {
- Svc.Prefs.set("client.name", value);
- // Update the registration in the background.
- fxAccounts.updateDeviceRegistration().catch(error => {
- this._log.warn("failed to update fxa device registration", error);
- });
- },
+ let localName = Svc.Prefs.get("client.name", "");
+ if (localName != "")
+ return localName;
- get localType() {
- return Utils.getDeviceType();
- },
- set localType(value) {
- Svc.Prefs.set("client.type", value);
+ return this.localName = Utils.getDefaultDeviceName();
},
+ set localName(value) Svc.Prefs.set("client.name", value),
- getClientName(id) {
- if (id == this.localID) {
- return this.localName;
- }
- let client = this._store._remoteClients[id];
- return client ? client.name : "";
- },
-
- getClientFxaDeviceId(id) {
- if (this._store._remoteClients[id]) {
- return this._store._remoteClients[id].fxaDeviceId;
- }
- return null;
- },
+ get localType() Svc.Prefs.get("client.type", "desktop"),
+ set localType(value) Svc.Prefs.set("client.type", value),
isMobile: function isMobile(id) {
if (this._store._remoteClients[id])
- return this._store._remoteClients[id].type == DEVICE_TYPE_MOBILE;
+ return this._store._remoteClients[id].type == "mobile";
return false;
},
- _readCommands() {
- let cb = Async.makeSpinningCallback();
- Utils.jsonLoad("commands", this, commands => cb(null, commands));
- return cb.wait() || {};
- },
-
- /**
- * Low level function, do not use directly (use _addClientCommand instead).
- */
- _saveCommands(commands) {
- let cb = Async.makeSpinningCallback();
- Utils.jsonSave("commands", this, commands, error => {
- if (error) {
- this._log.error("Failed to save JSON outgoing commands", error);
- }
- cb();
- });
- cb.wait();
- },
-
- _prepareCommandsForUpload() {
- let cb = Async.makeSpinningCallback();
- Utils.jsonMove("commands", "commands-syncing", this).catch(() => {}) // Ignore errors
- .then(() => {
- Utils.jsonLoad("commands-syncing", this, commands => cb(null, commands));
- });
- return cb.wait() || {};
- },
-
- _deleteUploadedCommands() {
- delete this._currentlySyncingCommands;
- Async.promiseSpinningly(
- Utils.jsonRemove("commands-syncing", this).catch(err => {
- this._log.error("Failed to delete syncing-commands file", err);
- })
- );
- },
-
- _addClientCommand(clientId, command) {
- const allCommands = this._readCommands();
- const clientCommands = allCommands[clientId] || [];
- if (hasDupeCommand(clientCommands, command)) {
- return;
- }
- allCommands[clientId] = clientCommands.concat(command);
- this._saveCommands(allCommands);
- },
-
_syncStartup: function _syncStartup() {
// Reupload new client record periodically.
if (Date.now() / 1000 - this.lastRecordUpload > CLIENTS_TTL_REFRESH) {
@@ -266,157 +136,9 @@ ClientEngine.prototype = {
SyncEngine.prototype._syncStartup.call(this);
},
- _processIncoming() {
- // Fetch all records from the server.
- this.lastSync = 0;
- this._incomingClients = {};
- try {
- SyncEngine.prototype._processIncoming.call(this);
- // Since clients are synced unconditionally, any records in the local store
- // that don't exist on the server must be for disconnected clients. Remove
- // them, so that we don't upload records with commands for clients that will
- // never see them. We also do this to filter out stale clients from the
- // tabs collection, since showing their list of tabs is confusing.
- for (let id in this._store._remoteClients) {
- if (!this._incomingClients[id]) {
- this._log.info(`Removing local state for deleted client ${id}`);
- this._removeRemoteClient(id);
- }
- }
- // Bug 1264498: Mobile clients don't remove themselves from the clients
- // collection when the user disconnects Sync, so we mark as stale clients
- // with the same name that haven't synced in over a week.
- // (Note we can't simply delete them, or we re-apply them next sync - see
- // bug 1287687)
- delete this._incomingClients[this.localID];
- let names = new Set([this.localName]);
- for (let id in this._incomingClients) {
- let record = this._store._remoteClients[id];
- if (!names.has(record.name)) {
- names.add(record.name);
- continue;
- }
- let remoteAge = AsyncResource.serverTime - this._incomingClients[id];
- if (remoteAge > STALE_CLIENT_REMOTE_AGE) {
- this._log.info(`Hiding stale client ${id} with age ${remoteAge}`);
- record.stale = true;
- }
- }
- } finally {
- this._incomingClients = null;
- }
- },
-
- _uploadOutgoing() {
- this._currentlySyncingCommands = this._prepareCommandsForUpload();
- const clientWithPendingCommands = Object.keys(this._currentlySyncingCommands);
- for (let clientId of clientWithPendingCommands) {
- if (this._store._remoteClients[clientId] || this.localID == clientId) {
- this._modified.set(clientId, 0);
- }
- }
- SyncEngine.prototype._uploadOutgoing.call(this);
- },
-
- _onRecordsWritten(succeeded, failed) {
- // Reconcile the status of the local records with what we just wrote on the
- // server
- for (let id of succeeded) {
- const commandChanges = this._currentlySyncingCommands[id];
- if (id == this.localID) {
- if (this.localCommands) {
- this.localCommands = this.localCommands.filter(command => !hasDupeCommand(commandChanges, command));
- }
- } else {
- const clientRecord = this._store._remoteClients[id];
- if (!commandChanges || !clientRecord) {
- // should be impossible, else we wouldn't have been writing it.
- this._log.warn("No command/No record changes for a client we uploaded");
- continue;
- }
- // fixup the client record, so our copy of _remoteClients matches what we uploaded.
- clientRecord.commands = this._store.createRecord(id);
- // we could do better and pass the reference to the record we just uploaded,
- // but this will do for now
- }
- }
-
- // Re-add failed commands
- for (let id of failed) {
- const commandChanges = this._currentlySyncingCommands[id];
- if (!commandChanges) {
- continue;
- }
- this._addClientCommand(id, commandChanges);
- }
-
- this._deleteUploadedCommands();
-
- // Notify other devices that their own client collection changed
- const idsToNotify = succeeded.reduce((acc, id) => {
- if (id == this.localID) {
- return acc;
- }
- const fxaDeviceId = this.getClientFxaDeviceId(id);
- return fxaDeviceId ? acc.concat(fxaDeviceId) : acc;
- }, []);
- if (idsToNotify.length > 0) {
- this._notifyCollectionChanged(idsToNotify);
- }
- },
-
- _notifyCollectionChanged(ids) {
- const message = {
- version: 1,
- command: "sync:collection_changed",
- data: {
- collections: ["clients"]
- }
- };
- fxAccounts.notifyDevices(ids, message, NOTIFY_TAB_SENT_TTL_SECS);
- },
-
- _syncFinish() {
- // Record histograms for our device types, and also write them to a pref
- // so non-histogram telemetry (eg, UITelemetry) has easy access to them.
- for (let [deviceType, count] of this.deviceTypes) {
- let hid;
- let prefName = this.name + ".devices.";
- switch (deviceType) {
- case "desktop":
- hid = "WEAVE_DEVICE_COUNT_DESKTOP";
- prefName += "desktop";
- break;
- case "mobile":
- hid = "WEAVE_DEVICE_COUNT_MOBILE";
- prefName += "mobile";
- break;
- default:
- this._log.warn(`Unexpected deviceType "${deviceType}" recording device telemetry.`);
- continue;
- }
- Services.telemetry.getHistogramById(hid).add(count);
- Svc.Prefs.set(prefName, count);
- }
- SyncEngine.prototype._syncFinish.call(this);
- },
-
- _reconcile: function _reconcile(item) {
- // Every incoming record is reconciled, so we use this to track the
- // contents of the collection on the server.
- this._incomingClients[item.id] = item.modified;
-
- if (!this._store.itemExists(item.id)) {
- return true;
- }
- // Clients are synced unconditionally, so we'll always have new records.
- // Unfortunately, this will cause the scheduler to use the immediate sync
- // interval for the multi-device case, instead of the active interval. We
- // work around this by updating the record during reconciliation, and
- // returning false to indicate that the record doesn't need to be applied
- // later.
- this._store.update(item);
- return false;
+ // Always process incoming items because they might have commands
+ _reconcile: function _reconcile() {
+ return true;
},
// Treat reset the same as wiping for locally cached clients
@@ -426,13 +148,7 @@ ClientEngine.prototype = {
_wipeClient: function _wipeClient() {
SyncEngine.prototype._resetClient.call(this);
- delete this.localCommands;
this._store.wipe();
- const logRemoveError = err => this._log.warn("Could not delete json file", err);
- Async.promiseSpinningly(
- Utils.jsonRemove("commands", this).catch(logRemoveError)
- .then(Utils.jsonRemove("commands-syncing", this).catch(logRemoveError))
- );
},
removeClientData: function removeClientData() {
@@ -471,6 +187,14 @@ ClientEngine.prototype = {
},
/**
+ * Remove any commands for the local client and mark it for upload.
+ */
+ clearCommands: function clearCommands() {
+ delete this.localCommands;
+ this._tracker.addChangedID(this.localID);
+ },
+
+ /**
* Sends a command+args pair to a specific client.
*
* @param command Command string
@@ -484,17 +208,30 @@ ClientEngine.prototype = {
if (!client) {
throw new Error("Unknown remote client ID: '" + clientId + "'.");
}
- if (client.stale) {
- throw new Error("Stale remote client ID: '" + clientId + "'.");
- }
+
+ // notDupe compares two commands and returns if they are not equal.
+ let notDupe = function(other) {
+ return other.command != command || !Utils.deepEquals(other.args, args);
+ };
let action = {
command: command,
args: args,
};
+ if (!client.commands) {
+ client.commands = [action];
+ }
+ // Add the new action if there are no duplicates.
+ else if (client.commands.every(notDupe)) {
+ client.commands.push(action);
+ }
+ // It must be a dupe. Skip.
+ else {
+ return;
+ }
+
this._log.trace("Client " + clientId + " got a new action: " + [command, args]);
- this._addClientCommand(clientId, action);
this._tracker.addChangedID(clientId);
},
@@ -505,17 +242,13 @@ ClientEngine.prototype = {
*/
processIncomingCommands: function processIncomingCommands() {
return this._notify("clients:process-commands", "", function() {
- if (!this.localCommands) {
- return true;
- }
+ let commands = this.localCommands;
- const clearedCommands = this._readCommands()[this.localID];
- const commands = this.localCommands.filter(command => !hasDupeCommand(clearedCommands, command));
+ // Immediately clear out the commands as we've got them locally.
+ this.clearCommands();
- let URIsToDisplay = [];
// Process each command in order.
- for (let rawCommand of commands) {
- let {command, args} = rawCommand;
+ for each (let {command, args} in commands) {
this._log.debug("Processing command: " + command + "(" + args + ")");
let engines = [args[0]];
@@ -536,20 +269,12 @@ ClientEngine.prototype = {
this.service.logout();
return false;
case "displayURI":
- let [uri, clientId, title] = args;
- URIsToDisplay.push({ uri, clientId, title });
+ this._handleDisplayURI.apply(this, args);
break;
default:
this._log.debug("Received an unknown command: " + command);
break;
}
- // Add the command to the "cleared" commands list
- this._addClientCommand(this.localID, rawCommand)
- }
- this._tracker.addChangedID(this.localID);
-
- if (URIsToDisplay.length) {
- this._handleDisplayURIs(URIsToDisplay);
}
return true;
@@ -588,10 +313,8 @@ ClientEngine.prototype = {
if (clientId) {
this._sendCommandToClient(command, args, clientId);
} else {
- for (let [id, record] of Object.entries(this._store._remoteClients)) {
- if (!record.stale) {
- this._sendCommandToClient(command, args, id);
- }
+ for (let id in this._store._remoteClients) {
+ this._sendCommandToClient(command, args, id);
}
}
},
@@ -622,11 +345,11 @@ ClientEngine.prototype = {
},
/**
- * Handle a bunch of received 'displayURI' commands.
+ * Handle a single received 'displayURI' command.
*
- * Interested parties should observe the "weave:engine:clients:display-uris"
- * topic. The callback will receive an array as the subject parameter
- * containing objects with the following keys:
+ * Interested parties should observe the "weave:engine:clients:display-uri"
+ * topic. The callback will receive an object as the subject parameter with
+ * the following keys:
*
* uri URI (string) that is requested for display.
* clientId ID of client that sent the command.
@@ -634,24 +357,21 @@ ClientEngine.prototype = {
*
* The 'data' parameter to the callback will not be defined.
*
- * @param uris
- * An array containing URI objects to display
- * @param uris[].uri
+ * @param uri
* String URI that was received
- * @param uris[].clientId
+ * @param clientId
* ID of client that sent URI
- * @param uris[].title
+ * @param title
* String title of page that URI corresponds to. Older clients may not
* send this.
*/
- _handleDisplayURIs: function _handleDisplayURIs(uris) {
- Svc.Obs.notify("weave:engine:clients:display-uris", uris);
- },
+ _handleDisplayURI: function _handleDisplayURI(uri, clientId, title) {
+ this._log.info("Received a URI for display: " + uri + " (" + title +
+ ") from " + clientId);
- _removeRemoteClient(id) {
- delete this._store._remoteClients[id];
- this._tracker.removeChangedID(id);
- },
+ let subject = {uri: uri, client: clientId, title: title};
+ Svc.Obs.notify("weave:engine:clients:display-uri", subject);
+ }
};
function ClientStore(name, engine) {
@@ -660,48 +380,29 @@ function ClientStore(name, engine) {
ClientStore.prototype = {
__proto__: Store.prototype,
- _remoteClients: {},
-
create(record) {
- this.update(record);
+ this.update(record)
},
update: function update(record) {
- if (record.id == this.engine.localID) {
- // Only grab commands from the server; local name/type always wins
+ // Only grab commands from the server; local name/type always wins
+ if (record.id == this.engine.localID)
this.engine.localCommands = record.commands;
- } else {
+ else
this._remoteClients[record.id] = record.cleartext;
- }
},
createRecord: function createRecord(id, collection) {
let record = new ClientsRec(collection, id);
- const commandsChanges = this.engine._currentlySyncingCommands ?
- this.engine._currentlySyncingCommands[id] :
- [];
-
// Package the individual components into a record for the local client
if (id == this.engine.localID) {
- let cb = Async.makeSpinningCallback();
- fxAccounts.getDeviceId().then(id => cb(null, id), cb);
- try {
- record.fxaDeviceId = cb.wait();
- } catch(error) {
- this._log.warn("failed to get fxa device id", error);
- }
record.name = this.engine.localName;
record.type = this.engine.localType;
+ record.commands = this.engine.localCommands;
record.version = Services.appinfo.version;
record.protocols = SUPPORTED_PROTOCOL_VERSIONS;
- // Substract the commands we recorded that we've already executed
- if (commandsChanges && commandsChanges.length &&
- this.engine.localCommands && this.engine.localCommands.length) {
- record.commands = this.engine.localCommands.filter(command => !hasDupeCommand(commandsChanges, command));
- }
-
// Optional fields.
record.os = Services.appinfo.OS; // "Darwin"
record.appPackage = Services.appinfo.ID;
@@ -712,20 +413,6 @@ ClientStore.prototype = {
// record.formfactor = ""; // Bug 1100722
} else {
record.cleartext = this._remoteClients[id];
-
- // Add the commands we have to send
- if (commandsChanges && commandsChanges.length) {
- const recordCommands = record.cleartext.commands || [];
- const newCommands = commandsChanges.filter(command => !hasDupeCommand(recordCommands, command));
- record.cleartext.commands = recordCommands.concat(newCommands);
- }
-
- if (record.cleartext.stale) {
- // It's almost certainly a logic error for us to upload a record we
- // consider stale, so make log noise, but still remove the flag.
- this._log.error(`Preparing to upload record ${id} that we consider stale`);
- delete record.cleartext.stale;
- }
}
return record;
@@ -768,7 +455,7 @@ ClientsTracker.prototype = {
break;
case "weave:engine:stop-tracking":
if (this._enabled) {
- Svc.Prefs.ignore("client.name", this);
+ Svc.Prefs.ignore("clients.name", this);
this._enabled = false;
}
break;
diff --git a/services/sync/modules/engines/forms.js b/services/sync/modules/engines/forms.js
index 43f79d4f7..d26d57176 100644
--- a/services/sync/modules/engines/forms.js
+++ b/services/sync/modules/engines/forms.js
@@ -2,11 +2,11 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-this.EXPORTED_SYMBOLS = ['FormEngine', 'FormRec', 'FormValidator'];
+this.EXPORTED_SYMBOLS = ['FormEngine', 'FormRec'];
-var Cc = Components.classes;
-var Ci = Components.interfaces;
-var Cu = Components.utils;
+const Cc = Components.classes;
+const Ci = Components.interfaces;
+const Cu = Components.utils;
Cu.import("resource://gre/modules/XPCOMUtils.jsm");
Cu.import("resource://services-sync/engines.js");
@@ -14,10 +14,9 @@ Cu.import("resource://services-sync/record.js");
Cu.import("resource://services-common/async.js");
Cu.import("resource://services-sync/util.js");
Cu.import("resource://services-sync/constants.js");
-Cu.import("resource://services-sync/collection_validator.js");
Cu.import("resource://gre/modules/Log.jsm");
-const FORMS_TTL = 3 * 365 * 24 * 60 * 60; // Three years in seconds.
+const FORMS_TTL = 5184000; // 60 days
this.FormRec = function FormRec(collection, id) {
CryptoWrapper.call(this, collection, id);
@@ -31,30 +30,26 @@ FormRec.prototype = {
Utils.deferGetSet(FormRec, "cleartext", ["name", "value"]);
-var FormWrapper = {
+let FormWrapper = {
_log: Log.repository.getLogger("Sync.Engine.Forms"),
_getEntryCols: ["fieldname", "value"],
_guidCols: ["guid"],
- _promiseSearch: function(terms, searchData) {
- return new Promise(resolve => {
- let results = [];
- let callbacks = {
- handleResult(result) {
- results.push(result);
- },
- handleCompletion(reason) {
- resolve(results);
- }
- };
- Svc.FormHistory.search(terms, searchData, callbacks);
- })
- },
-
// Do a "sync" search by spinning the event loop until it completes.
_searchSpinningly: function(terms, searchData) {
- return Async.promiseSpinningly(this._promiseSearch(terms, searchData));
+ let results = [];
+ let cb = Async.makeSpinningCallback();
+ let callbacks = {
+ handleResult: function(result) {
+ results.push(result);
+ },
+ handleCompletion: function(reason) {
+ cb(null, results);
+ }
+ };
+ Svc.FormHistory.search(terms, searchData, callbacks);
+ return cb.wait();
},
_updateSpinningly: function(changes) {
@@ -114,9 +109,7 @@ FormEngine.prototype = {
syncPriority: 6,
- get prefName() {
- return "history";
- },
+ get prefName() "history",
_findDupe: function _findDupe(item) {
return FormWrapper.getGUID(item.name, item.value);
@@ -232,9 +225,7 @@ FormTracker.prototype = {
observe: function (subject, topic, data) {
Tracker.prototype.observe.call(this, subject, topic, data);
- if (this.ignoreAll) {
- return;
- }
+
switch (topic) {
case "satchel-storage-changed":
if (data == "formhistory-add" || data == "formhistory-remove") {
@@ -250,56 +241,3 @@ FormTracker.prototype = {
this.score += SCORE_INCREMENT_MEDIUM;
},
};
-
-
-class FormsProblemData extends CollectionProblemData {
- getSummary() {
- // We don't support syncing deleted form data, so "clientMissing" isn't a problem
- return super.getSummary().filter(entry =>
- entry.name !== "clientMissing");
- }
-}
-
-class FormValidator extends CollectionValidator {
- constructor() {
- super("forms", "id", ["name", "value"]);
- }
-
- emptyProblemData() {
- return new FormsProblemData();
- }
-
- getClientItems() {
- return FormWrapper._promiseSearch(["guid", "fieldname", "value"], {});
- }
-
- normalizeClientItem(item) {
- return {
- id: item.guid,
- guid: item.guid,
- name: item.fieldname,
- fieldname: item.fieldname,
- value: item.value,
- original: item,
- };
- }
-
- normalizeServerItem(item) {
- let res = Object.assign({
- guid: item.id,
- fieldname: item.name,
- original: item,
- }, item);
- // Missing `name` or `value` causes the getGUID call to throw
- if (item.name !== undefined && item.value !== undefined) {
- let guid = FormWrapper.getGUID(item.name, item.value);
- if (guid) {
- res.guid = guid;
- res.id = guid;
- res.duped = true;
- }
- }
-
- return res;
- }
-} \ No newline at end of file
diff --git a/services/sync/modules/engines/history.js b/services/sync/modules/engines/history.js
index 307d484c1..99ecb4506 100644
--- a/services/sync/modules/engines/history.js
+++ b/services/sync/modules/engines/history.js
@@ -4,10 +4,10 @@
this.EXPORTED_SYMBOLS = ['HistoryEngine', 'HistoryRec'];
-var Cc = Components.classes;
-var Ci = Components.interfaces;
-var Cu = Components.utils;
-var Cr = Components.results;
+const Cc = Components.classes;
+const Ci = Components.interfaces;
+const Cu = Components.utils;
+const Cr = Components.results;
const HISTORY_TTL = 5184000; // 60 days
@@ -44,25 +44,6 @@ HistoryEngine.prototype = {
applyIncomingBatchSize: HISTORY_STORE_BATCH_SIZE,
syncPriority: 7,
-
- _processIncoming: function (newitems) {
- // We want to notify history observers that a batch operation is underway
- // so they don't do lots of work for each incoming record.
- let observers = PlacesUtils.history.getObservers();
- function notifyHistoryObservers(notification) {
- for (let observer of observers) {
- try {
- observer[notification]();
- } catch (ex) { }
- }
- }
- notifyHistoryObservers("onBeginUpdateBatch");
- try {
- return SyncEngine.prototype._processIncoming.call(this, newitems);
- } finally {
- notifyHistoryObservers("onEndUpdateBatch");
- }
- },
};
function HistoryStore(name, engine) {
@@ -70,8 +51,7 @@ function HistoryStore(name, engine) {
// Explicitly nullify our references to our cached services so we don't leak
Svc.Obs.add("places-shutdown", function() {
- for (let query in this._stmts) {
- let stmt = this._stmts;
+ for each ([query, stmt] in Iterator(this._stmts)) {
stmt.finalize();
}
this._stmts = {};
@@ -105,7 +85,7 @@ HistoryStore.prototype = {
return this._getStmt(
"UPDATE moz_places " +
"SET guid = :guid " +
- "WHERE url_hash = hash(:page_url) AND url = :page_url");
+ "WHERE url = :page_url");
},
// Some helper functions to handle GUIDs
@@ -127,7 +107,7 @@ HistoryStore.prototype = {
return this._getStmt(
"SELECT guid " +
"FROM moz_places " +
- "WHERE url_hash = hash(:page_url) AND url = :page_url");
+ "WHERE url = :page_url");
},
_guidCols: ["guid"],
@@ -146,12 +126,12 @@ HistoryStore.prototype = {
},
get _visitStm() {
- return this._getStmt(`/* do not warn (bug 599936) */
- SELECT visit_type type, visit_date date
- FROM moz_historyvisits
- JOIN moz_places h ON h.id = place_id
- WHERE url_hash = hash(:url) AND url = :url
- ORDER BY date DESC LIMIT 20`);
+ return this._getStmt(
+ "/* do not warn (bug 599936) */ " +
+ "SELECT visit_type type, visit_date date " +
+ "FROM moz_historyvisits " +
+ "WHERE place_id = (SELECT id FROM moz_places WHERE url = :url) " +
+ "ORDER BY date DESC LIMIT 10");
},
_visitCols: ["date", "type"],
@@ -223,10 +203,7 @@ HistoryStore.prototype = {
} else {
shouldApply = this._recordToPlaceInfo(record);
}
- } catch (ex) {
- if (Async.isShutdownException(ex)) {
- throw ex;
- }
+ } catch(ex) {
failed.push(record.id);
shouldApply = false;
}
@@ -299,14 +276,14 @@ HistoryStore.prototype = {
if (!visit.date || typeof visit.date != "number") {
this._log.warn("Encountered record with invalid visit date: "
+ visit.date);
- continue;
+ throw "Visit has no date!";
}
- if (!visit.type ||
- !Object.values(PlacesUtils.history.TRANSITIONS).includes(visit.type)) {
- this._log.warn("Encountered record with invalid visit type: " +
- visit.type + "; ignoring.");
- continue;
+ if (!visit.type || !(visit.type >= PlacesUtils.history.TRANSITION_LINK &&
+ visit.type <= PlacesUtils.history.TRANSITION_FRAMED_LINK)) {
+ this._log.warn("Encountered record with invalid visit type: "
+ + visit.type);
+ throw "Invalid visit type!";
}
// Dates need to be integers.
@@ -317,7 +294,6 @@ HistoryStore.prototype = {
// overwritten.
continue;
}
-
visit.visitDate = visit.date;
visit.transitionType = visit.type;
k += 1;
@@ -369,9 +345,7 @@ HistoryStore.prototype = {
},
wipe: function HistStore_wipe() {
- let cb = Async.makeSyncCallback();
- PlacesUtils.history.clear().then(result => {cb(null, result)}, err => {cb(err)});
- return Async.waitForSyncCallback(cb);
+ PlacesUtils.history.removeAllPages();
}
};
diff --git a/services/sync/modules/engines/passwords.js b/services/sync/modules/engines/passwords.js
index 51db49a0a..994b59767 100644
--- a/services/sync/modules/engines/passwords.js
+++ b/services/sync/modules/engines/passwords.js
@@ -2,16 +2,14 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-this.EXPORTED_SYMBOLS = ['PasswordEngine', 'LoginRec', 'PasswordValidator'];
+this.EXPORTED_SYMBOLS = ['PasswordEngine', 'LoginRec'];
-var {classes: Cc, interfaces: Ci, utils: Cu} = Components;
+const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
Cu.import("resource://services-sync/record.js");
Cu.import("resource://services-sync/constants.js");
-Cu.import("resource://services-sync/collection_validator.js");
Cu.import("resource://services-sync/engines.js");
Cu.import("resource://services-sync/util.js");
-Cu.import("resource://services-common/async.js");
this.LoginRec = function LoginRec(collection, id) {
CryptoWrapper.call(this, collection, id);
@@ -24,7 +22,6 @@ LoginRec.prototype = {
Utils.deferGetSet(LoginRec, "cleartext", [
"hostname", "formSubmitURL",
"httpRealm", "username", "password", "usernameField", "passwordField",
- "timeCreated", "timePasswordChanged",
]);
@@ -70,10 +67,7 @@ PasswordEngine.prototype = {
Svc.Prefs.set("deletePwdFxA", true);
Svc.Prefs.reset("deletePwd"); // The old prefname we previously used.
} catch (ex) {
- if (Async.isShutdownException(ex)) {
- throw ex;
- }
- this._log.debug("Password deletes failed", ex);
+ this._log.debug("Password deletes failed: " + Utils.exceptionStr(ex));
}
}
},
@@ -89,7 +83,7 @@ PasswordEngine.prototype = {
this._store._sleep(0); // Yield back to main thread after synchronous operation.
// Look for existing logins that match the hostname, but ignore the password.
- for (let local of logins) {
+ for each (let local in logins) {
if (login.matches(local, true) && local instanceof Ci.nsILoginMetaInfo) {
return local.guid;
}
@@ -104,13 +98,6 @@ function PasswordStore(name, engine) {
PasswordStore.prototype = {
__proto__: Store.prototype,
- _newPropertyBag: function () {
- return Cc["@mozilla.org/hash-property-bag;1"].createInstance(Ci.nsIWritablePropertyBag2);
- },
-
- /**
- * Return an instance of nsILoginInfo (and, implicitly, nsILoginMetaInfo).
- */
_nsLoginInfoFromRecord: function (record) {
function nullUndefined(x) {
return (x == undefined) ? null : x;
@@ -131,21 +118,13 @@ PasswordStore.prototype = {
record.password,
record.usernameField,
record.passwordField);
-
info.QueryInterface(Ci.nsILoginMetaInfo);
info.guid = record.id;
- if (record.timeCreated) {
- info.timeCreated = record.timeCreated;
- }
- if (record.timePasswordChanged) {
- info.timePasswordChanged = record.timePasswordChanged;
- }
-
return info;
},
_getLoginFromGUID: function (id) {
- let prop = this._newPropertyBag();
+ let prop = Cc["@mozilla.org/hash-property-bag;1"].createInstance(Ci.nsIWritablePropertyBag2);
prop.setPropertyAsAUTF8String("guid", id);
let logins = Services.logins.searchLogins({}, prop);
@@ -190,7 +169,8 @@ PasswordStore.prototype = {
return;
}
- let prop = this._newPropertyBag();
+ let prop = Cc["@mozilla.org/hash-property-bag;1"]
+ .createInstance(Ci.nsIWritablePropertyBag2);
prop.setPropertyAsAUTF8String("guid", newID);
Services.logins.modifyLogin(oldLogin, prop);
@@ -217,11 +197,6 @@ PasswordStore.prototype = {
record.usernameField = login.usernameField;
record.passwordField = login.passwordField;
- // Optional fields.
- login.QueryInterface(Ci.nsILoginMetaInfo);
- record.timeCreated = login.timeCreated;
- record.timePasswordChanged = login.timePasswordChanged;
-
return record;
},
@@ -237,7 +212,8 @@ PasswordStore.prototype = {
try {
Services.logins.addLogin(login);
} catch(ex) {
- this._log.debug(`Adding record ${record.id} resulted in exception`, ex);
+ this._log.debug("Adding record " + record.id +
+ " resulted in exception " + Utils.exceptionStr(ex));
}
},
@@ -269,7 +245,9 @@ PasswordStore.prototype = {
try {
Services.logins.modifyLogin(loginItem, newinfo);
} catch(ex) {
- this._log.debug(`Modifying record ${record.id} resulted in exception; not modifying`, ex);
+ this._log.debug("Modifying record " + record.id +
+ " resulted in exception " + Utils.exceptionStr(ex) +
+ ". Not modifying.");
}
},
@@ -326,46 +304,3 @@ PasswordTracker.prototype = {
}
},
};
-
-class PasswordValidator extends CollectionValidator {
- constructor() {
- super("passwords", "id", [
- "hostname",
- "formSubmitURL",
- "httpRealm",
- "password",
- "passwordField",
- "username",
- "usernameField",
- ]);
- }
-
- getClientItems() {
- let logins = Services.logins.getAllLogins({});
- let syncHosts = Utils.getSyncCredentialsHosts()
- let result = logins.map(l => l.QueryInterface(Ci.nsILoginMetaInfo))
- .filter(l => !syncHosts.has(l.hostname));
- return Promise.resolve(result);
- }
-
- normalizeClientItem(item) {
- return {
- id: item.guid,
- guid: item.guid,
- hostname: item.hostname,
- formSubmitURL: item.formSubmitURL,
- httpRealm: item.httpRealm,
- password: item.password,
- passwordField: item.passwordField,
- username: item.username,
- usernameField: item.usernameField,
- original: item,
- }
- }
-
- normalizeServerItem(item) {
- return Object.assign({ guid: item.id }, item);
- }
-}
-
-
diff --git a/services/sync/modules/engines/prefs.js b/services/sync/modules/engines/prefs.js
index 9ceeb9ac6..82091d5b4 100644
--- a/services/sync/modules/engines/prefs.js
+++ b/services/sync/modules/engines/prefs.js
@@ -4,11 +4,11 @@
this.EXPORTED_SYMBOLS = ['PrefsEngine', 'PrefRec'];
-var Cc = Components.classes;
-var Ci = Components.interfaces;
-var Cu = Components.utils;
+const Cc = Components.classes;
+const Ci = Components.interfaces;
+const Cu = Components.utils;
-const PREF_SYNC_PREFS_PREFIX = "services.sync.prefs.sync.";
+const SYNC_PREFS_PREFIX = "services.sync.prefs.sync.";
Cu.import("resource://services-sync/engines.js");
Cu.import("resource://services-sync/record.js");
@@ -42,7 +42,6 @@ PrefsEngine.prototype = {
version: 2,
syncPriority: 1,
- allowSkippedRecord: false,
getChangedIDs: function () {
// No need for a proper timestamp (no conflict resolution needed).
@@ -88,45 +87,37 @@ PrefStore.prototype = {
_getSyncPrefs: function () {
let syncPrefs = Cc["@mozilla.org/preferences-service;1"]
.getService(Ci.nsIPrefService)
- .getBranch(PREF_SYNC_PREFS_PREFIX)
+ .getBranch(SYNC_PREFS_PREFIX)
.getChildList("", {});
// Also sync preferences that determine which prefs get synced.
- let controlPrefs = syncPrefs.map(pref => PREF_SYNC_PREFS_PREFIX + pref);
+ let controlPrefs = syncPrefs.map(pref => SYNC_PREFS_PREFIX + pref);
return controlPrefs.concat(syncPrefs);
},
_isSynced: function (pref) {
- return pref.startsWith(PREF_SYNC_PREFS_PREFIX) ||
- this._prefs.get(PREF_SYNC_PREFS_PREFIX + pref, false);
+ return pref.startsWith(SYNC_PREFS_PREFIX) ||
+ this._prefs.get(SYNC_PREFS_PREFIX + pref, false);
},
_getAllPrefs: function () {
let values = {};
- for (let pref of this._getSyncPrefs()) {
+ for each (let pref in this._getSyncPrefs()) {
if (this._isSynced(pref)) {
- // Missing and default prefs get the null value.
- values[pref] = this._prefs.isSet(pref) ? this._prefs.get(pref, null) : null;
+ // Missing prefs get the null value.
+ values[pref] = this._prefs.get(pref, null);
}
}
return values;
},
- _updateLightWeightTheme (themeID) {
- let themeObject = null;
- if (themeID) {
- themeObject = LightweightThemeManager.getUsedTheme(themeID);
- }
- LightweightThemeManager.currentTheme = themeObject;
- },
-
_setAllPrefs: function (values) {
- let selectedThemeIDPref = "lightweightThemes.selectedThemeID";
- let selectedThemeIDBefore = this._prefs.get(selectedThemeIDPref, null);
- let selectedThemeIDAfter = selectedThemeIDBefore;
+ let enabledPref = "lightweightThemes.isThemeSelected";
+ let enabledBefore = this._prefs.get(enabledPref, false);
+ let prevTheme = LightweightThemeManager.currentTheme;
// Update 'services.sync.prefs.sync.foo.pref' before 'foo.pref', otherwise
// _isSynced returns false when 'foo.pref' doesn't exist (e.g., on a new device).
- let prefs = Object.keys(values).sort(a => -a.indexOf(PREF_SYNC_PREFS_PREFIX));
+ let prefs = Object.keys(values).sort(a => -a.indexOf(SYNC_PREFS_PREFIX));
for (let pref of prefs) {
if (!this._isSynced(pref)) {
continue;
@@ -134,30 +125,26 @@ PrefStore.prototype = {
let value = values[pref];
- switch (pref) {
- // Some special prefs we don't want to set directly.
- case selectedThemeIDPref:
- selectedThemeIDAfter = value;
- break;
-
- // default is to just set the pref
- default:
- if (value == null) {
- // Pref has gone missing. The best we can do is reset it.
- this._prefs.reset(pref);
- } else {
- try {
- this._prefs.set(pref, value);
- } catch(ex) {
- this._log.trace("Failed to set pref: " + pref + ": " + ex);
- }
- }
+ // Pref has gone missing. The best we can do is reset it.
+ if (value == null) {
+ this._prefs.reset(pref);
+ continue;
}
+
+ try {
+ this._prefs.set(pref, value);
+ } catch(ex) {
+ this._log.trace("Failed to set pref: " + pref + ": " + ex);
+ }
}
- // Notify the lightweight theme manager if the selected theme has changed.
- if (selectedThemeIDBefore != selectedThemeIDAfter) {
- this._updateLightWeightTheme(selectedThemeIDAfter);
+ // Notify the lightweight theme manager of all the new values
+ let enabledNow = this._prefs.get(enabledPref, false);
+ if (enabledBefore && !enabledNow) {
+ LightweightThemeManager.currentTheme = null;
+ } else if (enabledNow && LightweightThemeManager.usedThemes[0] != prevTheme) {
+ LightweightThemeManager.currentTheme = null;
+ LightweightThemeManager.currentTheme = LightweightThemeManager.usedThemes[0];
}
},
@@ -261,8 +248,8 @@ PrefTracker.prototype = {
case "nsPref:changed":
// Trigger a sync for MULTI-DEVICE for a change that determines
// which prefs are synced or a regular pref change.
- if (data.indexOf(PREF_SYNC_PREFS_PREFIX) == 0 ||
- this._prefs.get(PREF_SYNC_PREFS_PREFIX + data, false)) {
+ if (data.indexOf(SYNC_PREFS_PREFIX) == 0 ||
+ this._prefs.get(SYNC_PREFS_PREFIX + data, false)) {
this.score += SCORE_INCREMENT_XLARGE;
this.modified = true;
this._log.trace("Preference " + data + " changed");
diff --git a/services/sync/modules/engines/tabs.js b/services/sync/modules/engines/tabs.js
index 45ece4a23..1fce737d2 100644
--- a/services/sync/modules/engines/tabs.js
+++ b/services/sync/modules/engines/tabs.js
@@ -4,7 +4,7 @@
this.EXPORTED_SYMBOLS = ["TabEngine", "TabSetRecord"];
-var {classes: Cc, interfaces: Ci, utils: Cu} = Components;
+const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
const TABS_TTL = 604800; // 7 days.
const TAB_ENTRIES_LIMIT = 25; // How many URLs to include in tab history.
@@ -43,11 +43,6 @@ TabEngine.prototype = {
_storeObj: TabStore,
_trackerObj: TabTracker,
_recordObj: TabSetRecord,
- // A flag to indicate if we have synced in this session. This is to help
- // consumers of remote tabs that may want to differentiate between "I've an
- // empty tab list as I haven't yet synced" vs "I've an empty tab list
- // as there really are no tabs"
- hasSyncedThisSession: false,
syncPriority: 3,
@@ -72,7 +67,6 @@ TabEngine.prototype = {
SyncEngine.prototype._resetClient.call(this);
this._store.wipe();
this._tracker.modified = true;
- this.hasSyncedThisSession = false;
},
removeClientData: function () {
@@ -100,12 +94,7 @@ TabEngine.prototype = {
}
return SyncEngine.prototype._reconcile.call(this, item);
- },
-
- _syncFinish() {
- this.hasSyncedThisSession = true;
- return SyncEngine.prototype._syncFinish.call(this);
- },
+ }
};
@@ -145,7 +134,7 @@ TabStore.prototype = {
}
for (let tab of win.gBrowser.tabs) {
- let tabState = this.getTabState(tab);
+ tabState = this.getTabState(tab);
// Make sure there are history entries to look at.
if (!tabState || !tabState.entries.length) {
@@ -165,11 +154,6 @@ TabStore.prototype = {
continue;
}
- if (current.url.length >= (MAX_UPLOAD_BYTES - 1000)) {
- this._log.trace("Skipping over-long URL.");
- continue;
- }
-
// The element at `index` is the current page. Previous URLs were
// previously visited URLs; subsequent URLs are in the 'forward' stack,
// which we can't represent in Sync, so we truncate here.
@@ -189,9 +173,7 @@ TabStore.prototype = {
allTabs.push({
title: current.title || "",
urlHistory: urls,
- icon: tabState.image ||
- (tabState.attributes && tabState.attributes.image) ||
- "",
+ icon: tabState.attributes && tabState.attributes.image || "",
lastUsed: Math.floor((tabState.lastAccessed || 0) / 1000),
});
}
@@ -265,9 +247,27 @@ TabStore.prototype = {
create: function (record) {
this._log.debug("Adding remote tabs from " + record.clientName);
- this._remoteClients[record.id] = Object.assign({}, record.cleartext, {
- lastModified: record.modified
- });
+ this._remoteClients[record.id] = record.cleartext;
+
+ // Lose some precision, but that's good enough (seconds).
+ let roundModify = Math.floor(record.modified / 1000);
+ let notifyState = Svc.Prefs.get("notifyTabState");
+
+ // If there's no existing pref, save this first modified time.
+ if (notifyState == null) {
+ Svc.Prefs.set("notifyTabState", roundModify);
+ return;
+ }
+
+ // Don't change notifyState if it's already 0 (don't notify).
+ if (notifyState == 0) {
+ return;
+ }
+
+ // We must have gotten a new tab that isn't the same as last time.
+ if (notifyState != roundModify) {
+ Svc.Prefs.set("notifyTabState", 0);
+ }
},
update: function (record) {
@@ -302,14 +302,10 @@ TabTracker.prototype = {
_registerListenersForWindow: function (window) {
this._log.trace("Registering tab listeners in window");
- for (let topic of this._topics) {
+ for each (let topic in this._topics) {
window.addEventListener(topic, this.onTab, false);
}
window.addEventListener("unload", this._unregisterListeners, false);
- // If it's got a tab browser we can listen for things like navigation.
- if (window.gBrowser) {
- window.gBrowser.addProgressListener(this);
- }
},
_unregisterListeners: function (event) {
@@ -319,12 +315,9 @@ TabTracker.prototype = {
_unregisterListenersForWindow: function (window) {
this._log.trace("Removing tab listeners in window");
window.removeEventListener("unload", this._unregisterListeners, false);
- for (let topic of this._topics) {
+ for each (let topic in this._topics) {
window.removeEventListener(topic, this.onTab, false);
}
- if (window.gBrowser) {
- window.gBrowser.removeProgressListener(this);
- }
},
startTracking: function () {
@@ -380,14 +373,4 @@ TabTracker.prototype = {
this.score += SCORE_INCREMENT_SMALL;
}
},
-
- // web progress listeners.
- onLocationChange: function (webProgress, request, location, flags) {
- // We only care about top-level location changes which are not in the same
- // document.
- if (webProgress.isTopLevel &&
- ((flags & Ci.nsIWebProgressListener.LOCATION_CHANGE_SAME_DOCUMENT) == 0)) {
- this.modified = true;
- }
- },
};
diff --git a/services/sync/modules/healthreport.jsm b/services/sync/modules/healthreport.jsm
new file mode 100644
index 000000000..47161c095
--- /dev/null
+++ b/services/sync/modules/healthreport.jsm
@@ -0,0 +1,262 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+this.EXPORTED_SYMBOLS = [
+ "SyncProvider",
+];
+
+const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
+
+Cu.import("resource://gre/modules/Metrics.jsm", this);
+Cu.import("resource://gre/modules/Promise.jsm", this);
+Cu.import("resource://gre/modules/Services.jsm", this);
+Cu.import("resource://gre/modules/XPCOMUtils.jsm", this);
+
+const DAILY_LAST_NUMERIC_FIELD = {type: Metrics.Storage.FIELD_DAILY_LAST_NUMERIC};
+const DAILY_LAST_TEXT_FIELD = {type: Metrics.Storage.FIELD_DAILY_LAST_TEXT};
+const DAILY_COUNTER_FIELD = {type: Metrics.Storage.FIELD_DAILY_COUNTER};
+
+XPCOMUtils.defineLazyModuleGetter(this, "Weave",
+ "resource://services-sync/main.js");
+
+function SyncMeasurement1() {
+ Metrics.Measurement.call(this);
+}
+
+SyncMeasurement1.prototype = Object.freeze({
+ __proto__: Metrics.Measurement.prototype,
+
+ name: "sync",
+ version: 1,
+
+ fields: {
+ enabled: DAILY_LAST_NUMERIC_FIELD,
+ preferredProtocol: DAILY_LAST_TEXT_FIELD,
+ activeProtocol: DAILY_LAST_TEXT_FIELD,
+ syncStart: DAILY_COUNTER_FIELD,
+ syncSuccess: DAILY_COUNTER_FIELD,
+ syncError: DAILY_COUNTER_FIELD,
+ },
+});
+
+function SyncDevicesMeasurement1() {
+ Metrics.Measurement.call(this);
+}
+
+SyncDevicesMeasurement1.prototype = Object.freeze({
+ __proto__: Metrics.Measurement.prototype,
+
+ name: "devices",
+ version: 1,
+
+ fields: {},
+
+ shouldIncludeField: function (name) {
+ return true;
+ },
+
+ fieldType: function (name) {
+ return Metrics.Storage.FIELD_DAILY_COUNTER;
+ },
+});
+
+function SyncMigrationMeasurement1() {
+ Metrics.Measurement.call(this);
+}
+
+SyncMigrationMeasurement1.prototype = Object.freeze({
+ __proto__: Metrics.Measurement.prototype,
+
+ name: "migration",
+ version: 1,
+
+ fields: {
+ state: DAILY_LAST_TEXT_FIELD, // last "user" or "internal" state we saw for the day
+ accepted: DAILY_COUNTER_FIELD, // number of times user tried to start migration
+ declined: DAILY_COUNTER_FIELD, // number of times user closed nagging infobar
+ unlinked: DAILY_LAST_NUMERIC_FIELD, // did the user decline and unlink
+ },
+});
+
+this.SyncProvider = function () {
+ Metrics.Provider.call(this);
+};
+SyncProvider.prototype = Object.freeze({
+ __proto__: Metrics.Provider.prototype,
+
+ name: "org.mozilla.sync",
+
+ measurementTypes: [
+ SyncDevicesMeasurement1,
+ SyncMeasurement1,
+ SyncMigrationMeasurement1,
+ ],
+
+ _OBSERVERS: [
+ "weave:service:sync:start",
+ "weave:service:sync:finish",
+ "weave:service:sync:error",
+ "fxa-migration:state-changed",
+ "fxa-migration:internal-state-changed",
+ "fxa-migration:internal-telemetry",
+ ],
+
+ postInit: function () {
+ for (let o of this._OBSERVERS) {
+ Services.obs.addObserver(this, o, false);
+ }
+
+ return Promise.resolve();
+ },
+
+ onShutdown: function () {
+ for (let o of this._OBSERVERS) {
+ Services.obs.removeObserver(this, o);
+ }
+
+ return Promise.resolve();
+ },
+
+ observe: function (subject, topic, data) {
+ switch (topic) {
+ case "weave:service:sync:start":
+ case "weave:service:sync:finish":
+ case "weave:service:sync:error":
+ return this._observeSync(subject, topic, data);
+
+ case "fxa-migration:state-changed":
+ case "fxa-migration:internal-state-changed":
+ case "fxa-migration:internal-telemetry":
+ return this._observeMigration(subject, topic, data);
+ }
+ Cu.reportError("unexpected topic in sync healthreport provider: " + topic);
+ },
+
+ _observeSync: function (subject, topic, data) {
+ let field;
+ switch (topic) {
+ case "weave:service:sync:start":
+ field = "syncStart";
+ break;
+
+ case "weave:service:sync:finish":
+ field = "syncSuccess";
+ break;
+
+ case "weave:service:sync:error":
+ field = "syncError";
+ break;
+
+ default:
+ Cu.reportError("unexpected sync topic in sync healthreport provider: " + topic);
+ return;
+ }
+
+ let m = this.getMeasurement(SyncMeasurement1.prototype.name,
+ SyncMeasurement1.prototype.version);
+ return this.enqueueStorageOperation(function recordSyncEvent() {
+ return m.incrementDailyCounter(field);
+ });
+ },
+
+ _observeMigration: function(subject, topic, data) {
+ switch (topic) {
+ case "fxa-migration:state-changed":
+ case "fxa-migration:internal-state-changed": {
+ // We record both "user" and "internal" states in the same field. This
+ // works for us as user state is always null when there is an internal
+ // state.
+ if (!data) {
+ return; // we don't count the |null| state
+ }
+ let m = this.getMeasurement(SyncMigrationMeasurement1.prototype.name,
+ SyncMigrationMeasurement1.prototype.version);
+ return this.enqueueStorageOperation(function() {
+ return m.setDailyLastText("state", data);
+ });
+ }
+
+ case "fxa-migration:internal-telemetry": {
+ // |data| is our field name.
+ let m = this.getMeasurement(SyncMigrationMeasurement1.prototype.name,
+ SyncMigrationMeasurement1.prototype.version);
+ return this.enqueueStorageOperation(function() {
+ switch (data) {
+ case "accepted":
+ case "declined":
+ return m.incrementDailyCounter(data);
+ case "unlinked":
+ return m.setDailyLastNumeric(data, 1);
+ default:
+ Cu.reportError("Unexpected migration field in sync healthreport provider: " + data);
+ return Promise.resolve();
+ }
+ });
+ }
+
+ default:
+ Cu.reportError("unexpected migration topic in sync healthreport provider: " + topic);
+ return;
+ }
+ },
+
+ collectDailyData: function () {
+ return this.storage.enqueueTransaction(this._populateDailyData.bind(this));
+ },
+
+ _populateDailyData: function* () {
+ let m = this.getMeasurement(SyncMeasurement1.prototype.name,
+ SyncMeasurement1.prototype.version);
+
+ let svc = Cc["@mozilla.org/weave/service;1"]
+ .getService(Ci.nsISupports)
+ .wrappedJSObject;
+
+ let enabled = svc.enabled;
+ yield m.setDailyLastNumeric("enabled", enabled ? 1 : 0);
+
+ // preferredProtocol is constant and only changes as the client
+ // evolves.
+ yield m.setDailyLastText("preferredProtocol", "1.5");
+
+ let protocol = svc.fxAccountsEnabled ? "1.5" : "1.1";
+ yield m.setDailyLastText("activeProtocol", protocol);
+
+ if (!enabled) {
+ return;
+ }
+
+ // Before grabbing more information, be sure the Sync service
+ // is fully initialized. This has the potential to initialize
+ // Sync on the spot. This may be undesired if Sync appears to
+ // be enabled but it really isn't. That responsibility should
+ // be up to svc.enabled to not return false positives, however.
+ yield svc.whenLoaded();
+
+ if (Weave.Status.service != Weave.STATUS_OK) {
+ return;
+ }
+
+ // Device types are dynamic. So we need to dynamically create fields if
+ // they don't exist.
+ let dm = this.getMeasurement(SyncDevicesMeasurement1.prototype.name,
+ SyncDevicesMeasurement1.prototype.version);
+ let devices = Weave.Service.clientsEngine.deviceTypes;
+ for (let [field, count] of devices) {
+ let hasField = this.storage.hasFieldFromMeasurement(dm.id, field,
+ this.storage.FIELD_DAILY_LAST_NUMERIC);
+ let fieldID;
+ if (hasField) {
+ fieldID = this.storage.fieldIDFromMeasurement(dm.id, field);
+ } else {
+ fieldID = yield this.storage.registerField(dm.id, field,
+ this.storage.FIELD_DAILY_LAST_NUMERIC);
+ }
+
+ yield this.storage.setDailyLastNumericFromFieldID(fieldID, count);
+ }
+ },
+});
diff --git a/services/sync/modules/identity.js b/services/sync/modules/identity.js
index b4da8c0bb..2bee13b5b 100644
--- a/services/sync/modules/identity.js
+++ b/services/sync/modules/identity.js
@@ -6,14 +6,13 @@
this.EXPORTED_SYMBOLS = ["IdentityManager"];
-var {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
+const {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
Cu.import("resource://gre/modules/XPCOMUtils.jsm");
Cu.import("resource://gre/modules/Promise.jsm");
Cu.import("resource://services-sync/constants.js");
Cu.import("resource://gre/modules/Log.jsm");
Cu.import("resource://services-sync/util.js");
-Cu.import("resource://services-common/async.js");
// Lazy import to prevent unnecessary load on startup.
for (let symbol of ["BulkKeyBundle", "SyncKeyBundle"]) {
@@ -85,14 +84,18 @@ IdentityManager.prototype = {
_syncKeyBundle: null,
/**
- * Initialize the identity provider.
+ * Initialize the identity provider. Returns a promise that is resolved
+ * when initialization is complete and the provider can be queried for
+ * its state
*/
initialize: function() {
// Nothing to do for this identity provider.
+ return Promise.resolve();
},
finalize: function() {
// Nothing to do for this identity provider.
+ return Promise.resolve();
},
/**
@@ -111,6 +114,14 @@ IdentityManager.prototype = {
return Promise.resolve();
},
+ /**
+ * Indicates if the identity manager is still initializing
+ */
+ get readyToAuthenticate() {
+ // We initialize in a fully sync manner, so we are always finished.
+ return true;
+ },
+
get account() {
return Svc.Prefs.get("account", this.username);
},
@@ -195,7 +206,7 @@ IdentityManager.prototype = {
return null;
}
- for (let login of this._getLogins(PWDMGR_PASSWORD_REALM)) {
+ for each (let login in this._getLogins(PWDMGR_PASSWORD_REALM)) {
if (login.username.toLowerCase() == username) {
// It should already be UTF-8 encoded, but we don't take any chances.
this._basicPassword = Utils.encodeUTF8(login.password);
@@ -249,7 +260,7 @@ IdentityManager.prototype = {
return null;
}
- for (let login of this._getLogins(PWDMGR_PASSPHRASE_REALM)) {
+ for each (let login in this._getLogins(PWDMGR_PASSPHRASE_REALM)) {
if (login.username.toLowerCase() == username) {
this._syncKey = login.password;
}
@@ -326,7 +337,7 @@ IdentityManager.prototype = {
try {
this._syncKeyBundle = new SyncKeyBundle(this.username, this.syncKey);
} catch (ex) {
- this._log.warn("Failed to create sync bundle", ex);
+ this._log.warn(Utils.exceptionStr(ex));
return null;
}
}
@@ -400,7 +411,7 @@ IdentityManager.prototype = {
this._setLogin(PWDMGR_PASSWORD_REALM, this.username,
this._basicPassword);
} else {
- for (let login of this._getLogins(PWDMGR_PASSWORD_REALM)) {
+ for each (let login in this._getLogins(PWDMGR_PASSWORD_REALM)) {
Services.logins.removeLogin(login);
}
}
@@ -412,7 +423,7 @@ IdentityManager.prototype = {
if (this._syncKey) {
this._setLogin(PWDMGR_PASSPHRASE_REALM, this.username, this._syncKey);
} else {
- for (let login of this._getLogins(PWDMGR_PASSPHRASE_REALM)) {
+ for each (let login in this._getLogins(PWDMGR_PASSPHRASE_REALM)) {
Services.logins.removeLogin(login);
}
}
@@ -447,9 +458,6 @@ IdentityManager.prototype = {
try {
service.recordManager.get(service.storageURL + "meta/fxa_credentials");
} catch (ex) {
- if (Async.isShutdownException(ex)) {
- throw ex;
- }
this._log.warn("Failed to pre-fetch the migration sentinel", ex);
}
},
@@ -469,7 +477,7 @@ IdentityManager.prototype = {
*/
_setLogin: function _setLogin(realm, username, password) {
let exists = false;
- for (let login of this._getLogins(realm)) {
+ for each (let login in this._getLogins(realm)) {
if (login.username == username && login.password == password) {
exists = true;
} else {
@@ -505,7 +513,7 @@ IdentityManager.prototype = {
deleteSyncCredentials: function deleteSyncCredentials() {
for (let host of this._getSyncCredentialsHosts()) {
let logins = Services.logins.findLogins({}, host, "", "");
- for (let login of logins) {
+ for each (let login in logins) {
Services.logins.removeLogin(login);
}
}
@@ -593,13 +601,4 @@ IdentityManager.prototype = {
// Do nothing for Sync 1.1.
return {accepted: true};
},
-
- // Tell Sync what the login status should be if it saw a 401 fetching
- // info/collections as part of login verification (typically immediately
- // after login.)
- // In our case it means an authoritative "password is incorrect".
- loginStatusFromVerification404() {
- return LOGIN_FAILED_LOGIN_REJECTED;
- }
-
};
diff --git a/services/sync/modules/jpakeclient.js b/services/sync/modules/jpakeclient.js
index 625dc91b6..10f405371 100644
--- a/services/sync/modules/jpakeclient.js
+++ b/services/sync/modules/jpakeclient.js
@@ -4,7 +4,7 @@
this.EXPORTED_SYMBOLS = ["JPAKEClient", "SendCredentialsController"];
-var {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components;
+const {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components;
Cu.import("resource://gre/modules/Log.jsm");
Cu.import("resource://services-common/rest.js");
@@ -281,7 +281,8 @@ JPAKEClient.prototype = {
let rng = Cc["@mozilla.org/security/random-generator;1"]
.createInstance(Ci.nsIRandomGenerator);
let bytes = rng.generateRandomBytes(JPAKE_LENGTH_CLIENTID / 2);
- this._clientID = bytes.map(byte => ("0" + byte.toString(16)).slice(-2)).join("");
+ this._clientID = [("0" + byte.toString(16)).slice(-2)
+ for each (byte in bytes)].join("");
},
_createSecret: function _createSecret() {
@@ -290,7 +291,8 @@ JPAKEClient.prototype = {
let rng = Cc["@mozilla.org/security/random-generator;1"]
.createInstance(Ci.nsIRandomGenerator);
let bytes = rng.generateRandomBytes(JPAKE_LENGTH_SECRET);
- return bytes.map(byte => key[Math.floor(byte * key.length / 256)]).join("");
+ return [key[Math.floor(byte * key.length / 256)]
+ for each (byte in bytes)].join("");
},
_newRequest: function _newRequest(uri) {
diff --git a/services/sync/modules/keys.js b/services/sync/modules/keys.js
index b93de7f31..bf909bdc2 100644
--- a/services/sync/modules/keys.js
+++ b/services/sync/modules/keys.js
@@ -9,7 +9,7 @@ this.EXPORTED_SYMBOLS = [
"SyncKeyBundle"
];
-var {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
+const {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
Cu.import("resource://services-sync/constants.js");
Cu.import("resource://gre/modules/Log.jsm");
diff --git a/services/sync/modules/main.js b/services/sync/modules/main.js
index e8e705e72..488a2594b 100644
--- a/services/sync/modules/main.js
+++ b/services/sync/modules/main.js
@@ -6,7 +6,7 @@ this.EXPORTED_SYMBOLS = ['Weave'];
this.Weave = {};
Components.utils.import("resource://services-sync/constants.js", Weave);
-var lazies = {
+let lazies = {
"jpakeclient.js": ["JPAKEClient", "SendCredentialsController"],
"notifications.js": ["Notifications", "Notification", "NotificationButton"],
"service.js": ["Service"],
@@ -15,14 +15,12 @@ var lazies = {
};
function lazyImport(module, dest, props) {
- function getter(prop) {
- return function() {
- let ns = {};
- Components.utils.import(module, ns);
- delete dest[prop];
- return dest[prop] = ns[prop];
- };
- }
+ function getter(prop) function() {
+ let ns = {};
+ Components.utils.import(module, ns);
+ delete dest[prop];
+ return dest[prop] = ns[prop];
+ };
props.forEach(function (prop) { dest.__defineGetter__(prop, getter(prop)); });
}
diff --git a/services/sync/modules/notifications.js b/services/sync/modules/notifications.js
index 72187a4ce..1ee24f2cd 100644
--- a/services/sync/modules/notifications.js
+++ b/services/sync/modules/notifications.js
@@ -4,10 +4,10 @@
this.EXPORTED_SYMBOLS = ["Notifications", "Notification", "NotificationButton"];
-var Cc = Components.classes;
-var Ci = Components.interfaces;
-var Cr = Components.results;
-var Cu = Components.utils;
+const Cc = Components.classes;
+const Ci = Components.interfaces;
+const Cr = Components.results;
+const Cu = Components.utils;
Cu.import("resource://services-common/observers.js");
Cu.import("resource://gre/modules/Log.jsm");
diff --git a/services/sync/modules/policies.js b/services/sync/modules/policies.js
index a3933426d..d799cb235 100644
--- a/services/sync/modules/policies.js
+++ b/services/sync/modules/policies.js
@@ -7,26 +7,16 @@ this.EXPORTED_SYMBOLS = [
"SyncScheduler",
];
-var {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
+const {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
Cu.import("resource://gre/modules/Log.jsm");
Cu.import("resource://services-sync/constants.js");
Cu.import("resource://services-sync/engines.js");
Cu.import("resource://services-sync/util.js");
Cu.import("resource://services-common/logmanager.js");
-Cu.import("resource://services-common/async.js");
XPCOMUtils.defineLazyModuleGetter(this, "Status",
"resource://services-sync/status.js");
-XPCOMUtils.defineLazyModuleGetter(this, "AddonManager",
- "resource://gre/modules/AddonManager.jsm");
-
-// Get the value for an interval that's stored in preferences. To save users
-// from themselves (and us from them!) the minimum time they can specify
-// is 60s.
-function getThrottledIntervalPreference(prefName) {
- return Math.max(Svc.Prefs.get(prefName), 60) * 1000;
-}
this.SyncScheduler = function SyncScheduler(service) {
this.service = service;
@@ -55,12 +45,12 @@ SyncScheduler.prototype = {
let part = service.fxAccountsEnabled ? "fxa" : "sync11";
let prefSDInterval = "scheduler." + part + ".singleDeviceInterval";
- this.singleDeviceInterval = getThrottledIntervalPreference(prefSDInterval);
+ this.singleDeviceInterval = Svc.Prefs.get(prefSDInterval) * 1000;
- this.idleInterval = getThrottledIntervalPreference("scheduler.idleInterval");
- this.activeInterval = getThrottledIntervalPreference("scheduler.activeInterval");
- this.immediateInterval = getThrottledIntervalPreference("scheduler.immediateInterval");
- this.eolInterval = getThrottledIntervalPreference("scheduler.eolInterval");
+ this.idleInterval = Svc.Prefs.get("scheduler.idleInterval") * 1000;
+ this.activeInterval = Svc.Prefs.get("scheduler.activeInterval") * 1000;
+ this.immediateInterval = Svc.Prefs.get("scheduler.immediateInterval") * 1000;
+ this.eolInterval = Svc.Prefs.get("scheduler.eolInterval") * 1000;
// A user is non-idle on startup by default.
this.idle = false;
@@ -71,40 +61,20 @@ SyncScheduler.prototype = {
},
// nextSync is in milliseconds, but prefs can't hold that much
- get nextSync() {
- return Svc.Prefs.get("nextSync", 0) * 1000;
- },
- set nextSync(value) {
- Svc.Prefs.set("nextSync", Math.floor(value / 1000));
- },
+ get nextSync() Svc.Prefs.get("nextSync", 0) * 1000,
+ set nextSync(value) Svc.Prefs.set("nextSync", Math.floor(value / 1000)),
- get syncInterval() {
- return Svc.Prefs.get("syncInterval", this.singleDeviceInterval);
- },
- set syncInterval(value) {
- Svc.Prefs.set("syncInterval", value);
- },
+ get syncInterval() Svc.Prefs.get("syncInterval", this.singleDeviceInterval),
+ set syncInterval(value) Svc.Prefs.set("syncInterval", value),
- get syncThreshold() {
- return Svc.Prefs.get("syncThreshold", SINGLE_USER_THRESHOLD);
- },
- set syncThreshold(value) {
- Svc.Prefs.set("syncThreshold", value);
- },
+ get syncThreshold() Svc.Prefs.get("syncThreshold", SINGLE_USER_THRESHOLD),
+ set syncThreshold(value) Svc.Prefs.set("syncThreshold", value),
- get globalScore() {
- return Svc.Prefs.get("globalScore", 0);
- },
- set globalScore(value) {
- Svc.Prefs.set("globalScore", value);
- },
+ get globalScore() Svc.Prefs.get("globalScore", 0),
+ set globalScore(value) Svc.Prefs.set("globalScore", value),
- get numClients() {
- return Svc.Prefs.get("numClients", 0);
- },
- set numClients(value) {
- Svc.Prefs.set("numClients", value);
- },
+ get numClients() Svc.Prefs.get("numClients", 0),
+ set numClients(value) Svc.Prefs.set("numClients", value),
init: function init() {
this._log.level = Log.Level[Svc.Prefs.get("log.logger.service.main")];
@@ -249,10 +219,7 @@ SyncScheduler.prototype = {
this.setDefaults();
try {
Svc.Idle.removeIdleObserver(this, Svc.Prefs.get("scheduler.idleTime"));
- } catch (ex) {
- if (ex.result != Cr.NS_ERROR_FAILURE) {
- throw ex;
- }
+ } catch (ex if (ex.result == Cr.NS_ERROR_FAILURE)) {
// In all likelihood we didn't have an idle observer registered yet.
// It's all good.
}
@@ -285,11 +252,10 @@ SyncScheduler.prototype = {
case "wake_notification":
this._log.debug("Woke from sleep.");
Utils.nextTick(() => {
- // Trigger a sync if we have multiple clients. We give it 5 seconds
- // incase the network is still in the process of coming back up.
+ // Trigger a sync if we have multiple clients.
if (this.numClients > 1) {
- this._log.debug("More than 1 client. Will sync in 5s.");
- this.scheduleNextSync(5000);
+ this._log.debug("More than 1 client. Syncing.");
+ this.scheduleNextSync(0);
}
});
break;
@@ -531,6 +497,45 @@ SyncScheduler.prototype = {
this.syncTimer.clear();
},
+ /**
+ * Prevent new syncs from starting. This is used by the FxA migration code
+ * where we can't afford to have a sync start partway through the migration.
+ * To handle the edge-case of a sync starting and not stopping, we store
+ * this state in a pref, so on the next startup we remain blocked (and thus
+ * sync will never start) so the migration can complete.
+ *
+ * As a safety measure, we only block for some period of time, and after
+ * that it will automatically unblock. This ensures that if things go
+ * really pear-shaped and we never end up calling unblockSync() we haven't
+ * completely broken the world.
+ */
+ blockSync: function(until = null) {
+ if (!until) {
+ until = Date.now() + DEFAULT_BLOCK_PERIOD;
+ }
+ // until is specified in ms, but Prefs can't hold that much
+ Svc.Prefs.set("scheduler.blocked-until", Math.floor(until / 1000));
+ },
+
+ unblockSync: function() {
+ Svc.Prefs.reset("scheduler.blocked-until");
+ // the migration code should be ready to roll, so resume normal operations.
+ this.checkSyncStatus();
+ },
+
+ get isBlocked() {
+ let until = Svc.Prefs.get("scheduler.blocked-until");
+ if (until === undefined) {
+ return false;
+ }
+ if (until <= Math.floor(Date.now() / 1000)) {
+ // we were previously blocked but the time has expired.
+ Svc.Prefs.reset("scheduler.blocked-until");
+ return false;
+ }
+ // we remain blocked.
+ return true;
+ },
};
this.ErrorHandler = function ErrorHandler(service) {
@@ -570,10 +575,7 @@ ErrorHandler.prototype = {
root.level = Log.Level[Svc.Prefs.get("log.rootLogger")];
let logs = ["Sync", "FirefoxAccounts", "Hawk", "Common.TokenServerClient",
- "Sync.SyncMigration", "browserwindow.syncui",
- "Services.Common.RESTRequest", "Services.Common.RESTRequest",
- "BookmarkSyncUtils"
- ];
+ "Sync.SyncMigration"];
this._logManager = new LogManager(Svc.Prefs, logs, "sync");
},
@@ -590,25 +592,17 @@ ErrorHandler.prototype = {
this._log.debug(data + " failed to apply some records.");
}
break;
- case "weave:engine:sync:error": {
+ case "weave:engine:sync:error":
let exception = subject; // exception thrown by engine's sync() method
let engine_name = data; // engine name that threw the exception
this.checkServerError(exception);
Status.engines = [engine_name, exception.failureCode || ENGINE_UNKNOWN_FAIL];
- if (Async.isShutdownException(exception)) {
- this._log.debug(engine_name + " was interrupted due to the application shutting down");
- } else {
- this._log.debug(engine_name + " failed", exception);
- Services.telemetry.getKeyedHistogramById("WEAVE_ENGINE_SYNC_ERRORS")
- .add(engine_name);
- }
+ this._log.debug(engine_name + " failed: " + Utils.exceptionStr(exception));
break;
- }
case "weave:service:login:error":
- this._log.error("Sync encountered a login error");
- this.resetFileLog();
+ this.resetFileLog(this._logManager.REASON_ERROR);
if (this.shouldReportError()) {
this.notifyOnNextTick("weave:ui:login:error");
@@ -618,23 +612,12 @@ ErrorHandler.prototype = {
this.dontIgnoreErrors = false;
break;
- case "weave:service:sync:error": {
+ case "weave:service:sync:error":
if (Status.sync == CREDENTIALS_CHANGED) {
this.service.logout();
}
- let exception = subject;
- if (Async.isShutdownException(exception)) {
- // If we are shutting down we just log the fact, attempt to flush
- // the log file and get out of here!
- this._log.error("Sync was interrupted due to the application shutting down");
- this.resetFileLog();
- break;
- }
-
- // Not a shutdown related exception...
- this._log.error("Sync encountered an error", exception);
- this.resetFileLog();
+ this.resetFileLog(this._logManager.REASON_ERROR);
if (this.shouldReportError()) {
this.notifyOnNextTick("weave:ui:sync:error");
@@ -644,7 +627,6 @@ ErrorHandler.prototype = {
this.dontIgnoreErrors = false;
break;
- }
case "weave:service:sync:finish":
this._log.trace("Status.service is " + Status.service);
@@ -660,8 +642,8 @@ ErrorHandler.prototype = {
}
if (Status.service == SYNC_FAILED_PARTIAL) {
- this._log.error("Some engines did not sync correctly.");
- this.resetFileLog();
+ this._log.debug("Some engines did not sync correctly.");
+ this.resetFileLog(this._logManager.REASON_ERROR);
if (this.shouldReportError()) {
this.dontIgnoreErrors = false;
@@ -669,7 +651,7 @@ ErrorHandler.prototype = {
break;
}
} else {
- this.resetFileLog();
+ this.resetFileLog(this._logManager.REASON_SUCCESS);
}
this.dontIgnoreErrors = false;
this.notifyOnNextTick("weave:ui:sync:finish");
@@ -696,52 +678,22 @@ ErrorHandler.prototype = {
Utils.nextTick(this.service.sync, this.service);
},
- _dumpAddons: function _dumpAddons() {
- // Just dump the items that sync may be concerned with. Specifically,
- // active extensions that are not hidden.
- let addonPromise = new Promise(resolve => {
- try {
- AddonManager.getAddonsByTypes(["extension"], resolve);
- } catch (e) {
- this._log.warn("Failed to dump addons", e)
- resolve([])
- }
- });
-
- return addonPromise.then(addons => {
- let relevantAddons = addons.filter(x => x.isActive && !x.hidden);
- this._log.debug("Addons installed", relevantAddons.length);
- for (let addon of relevantAddons) {
- this._log.debug(" - ${name}, version ${version}, id ${id}", addon);
- }
- });
- },
-
/**
* Generate a log file for the sync that just completed
* and refresh the input & output streams.
+ *
+ * @param reason
+ * A constant from the LogManager that indicates the reason for the
+ * reset.
*/
- resetFileLog: function resetFileLog() {
- let onComplete = logType => {
+ resetFileLog: function resetFileLog(reason) {
+ let onComplete = () => {
Svc.Obs.notify("weave:service:reset-file-log");
this._log.trace("Notified: " + Date.now());
- if (logType == this._logManager.ERROR_LOG_WRITTEN) {
- Cu.reportError("Sync encountered an error - see about:sync-log for the log file.");
- }
};
-
- // If we're writing an error log, dump extensions that may be causing problems.
- let beforeResetLog;
- if (this._logManager.sawError) {
- beforeResetLog = this._dumpAddons();
- } else {
- beforeResetLog = Promise.resolve();
- }
// Note we do not return the promise here - the caller doesn't need to wait
// for this to complete.
- beforeResetLog
- .then(() => this._logManager.resetFileLog())
- .then(onComplete, onComplete);
+ this._logManager.resetFileLog(reason).then(onComplete, onComplete);
},
/**
@@ -775,9 +727,6 @@ ErrorHandler.prototype = {
}
},
- // A function to indicate if Sync errors should be "reported" - which in this
- // context really means "should be notify observers of an error" - but note
- // that since bug 1180587, no one is going to surface an error to the user.
shouldReportError: function shouldReportError() {
if (Status.login == MASTER_PASSWORD_LOCKED) {
this._log.trace("shouldReportError: false (master password locked).");
@@ -817,12 +766,8 @@ ErrorHandler.prototype = {
return false;
}
-
- let result = ([Status.login, Status.sync].indexOf(SERVER_MAINTENANCE) == -1 &&
- [Status.login, Status.sync].indexOf(LOGIN_FAILED_NETWORK_ERROR) == -1);
- this._log.trace("shouldReportError: ${result} due to login=${login}, sync=${sync}",
- {result, login: Status.login, sync: Status.sync});
- return result;
+ return ([Status.login, Status.sync].indexOf(SERVER_MAINTENANCE) == -1 &&
+ [Status.login, Status.sync].indexOf(LOGIN_FAILED_NETWORK_ERROR) == -1);
},
get currentAlertMode() {
@@ -925,7 +870,7 @@ ErrorHandler.prototype = {
case 401:
this.service.logout();
this._log.info("Got 401 response; resetting clusterURL.");
- this.service.clusterURL = null;
+ Svc.Prefs.reset("clusterURL");
let delay = 0;
if (Svc.Prefs.get("lastSyncReassigned")) {
diff --git a/services/sync/modules/record.js b/services/sync/modules/record.js
index f7a69d9ef..04ccd2dd2 100644
--- a/services/sync/modules/record.js
+++ b/services/sync/modules/record.js
@@ -10,10 +10,10 @@ this.EXPORTED_SYMBOLS = [
"Collection",
];
-var Cc = Components.classes;
-var Ci = Components.interfaces;
-var Cr = Components.results;
-var Cu = Components.utils;
+const Cc = Components.classes;
+const Ci = Components.interfaces;
+const Cr = Components.results;
+const Cu = Components.utils;
const CRYPTO_COLLECTION = "crypto";
const KEYS_WBO = "keys";
@@ -23,7 +23,6 @@ Cu.import("resource://services-sync/constants.js");
Cu.import("resource://services-sync/keys.js");
Cu.import("resource://services-sync/resource.js");
Cu.import("resource://services-sync/util.js");
-Cu.import("resource://services-common/async.js");
this.WBORecord = function WBORecord(collection, id) {
this.data = {};
@@ -86,7 +85,7 @@ WBORecord.prototype = {
toJSON: function toJSON() {
// Copy fields from data to be stringified, making sure payload is a string
let obj = {};
- for (let [key, val] of Object.entries(this.data))
+ for (let [key, val] in Iterator(this.data))
obj[key] = key == "payload" ? JSON.stringify(val) : val;
if (this.ttl)
obj.ttl = this.ttl;
@@ -196,9 +195,7 @@ CryptoWrapper.prototype = {
},
// The custom setter below masks the parent's getter, so explicitly call it :(
- get id() {
- return WBORecord.prototype.__lookupGetter__("id").call(this);
- },
+ get id() WBORecord.prototype.__lookupGetter__("id").call(this),
// Keep both plaintext and encrypted versions of the id to verify integrity
set id(val) {
@@ -238,11 +235,8 @@ RecordManager.prototype = {
record.deserialize(this.response);
return this.set(url, record);
- } catch (ex) {
- if (Async.isShutdownException(ex)) {
- throw ex;
- }
- this._log.debug("Failed to import record", ex);
+ } catch(ex) {
+ this._log.debug("Failed to import record: " + Utils.exceptionStr(ex));
return null;
}
},
@@ -281,10 +275,10 @@ RecordManager.prototype = {
* You can update this thing simply by giving it /info/collections. It'll
* use the last modified time to bring itself up to date.
*/
-this.CollectionKeyManager = function CollectionKeyManager(lastModified, default_, collections) {
- this.lastModified = lastModified || 0;
- this._default = default_ || null;
- this._collections = collections || {};
+this.CollectionKeyManager = function CollectionKeyManager() {
+ this.lastModified = 0;
+ this._collections = {};
+ this._default = null;
this._log = Log.repository.getLogger("Sync.CollectionKeyManager");
}
@@ -293,19 +287,6 @@ this.CollectionKeyManager = function CollectionKeyManager(lastModified, default_
// Note that the last modified time needs to be preserved.
CollectionKeyManager.prototype = {
- /**
- * Generate a new CollectionKeyManager that has the same attributes
- * as this one.
- */
- clone() {
- const newCollections = {};
- for (let c in this._collections) {
- newCollections[c] = this._collections[c];
- }
-
- return new CollectionKeyManager(this.lastModified, this._default, newCollections);
- },
-
// Return information about old vs new keys:
// * same: true if two collections are equal
// * changed: an array of collection names that changed.
@@ -328,7 +309,7 @@ CollectionKeyManager.prototype = {
// Return a sorted, unique array.
changed.sort();
let last;
- changed = changed.filter(x => (x != last) && (last = x));
+ changed = [x for each (x in changed) if ((x != last) && (last = x))];
return {same: changed.length == 0,
changed: changed};
},
@@ -374,15 +355,15 @@ CollectionKeyManager.prototype = {
/**
* Create a WBO for the current keys.
*/
- asWBO: function(collection, id) {
- return this._makeWBO(this._collections, this._default);
- },
+ asWBO: function(collection, id)
+ this._makeWBO(this._collections, this._default),
/**
* Compute a new default key, and new keys for any specified collections.
*/
newKeys: function(collections) {
- let newDefaultKeyBundle = this.newDefaultKeyBundle();
+ let newDefaultKey = new BulkKeyBundle(DEFAULT_KEYBUNDLE_NAME);
+ newDefaultKey.generateRandom();
let newColls = {};
if (collections) {
@@ -392,7 +373,7 @@ CollectionKeyManager.prototype = {
newColls[c] = b;
});
}
- return [newDefaultKeyBundle, newColls];
+ return [newDefaultKey, newColls];
},
/**
@@ -406,57 +387,6 @@ CollectionKeyManager.prototype = {
return this._makeWBO(newColls, newDefaultKey);
},
- /**
- * Create a new default key.
- *
- * @returns {BulkKeyBundle}
- */
- newDefaultKeyBundle() {
- const key = new BulkKeyBundle(DEFAULT_KEYBUNDLE_NAME);
- key.generateRandom();
- return key;
- },
-
- /**
- * Create a new default key and store it as this._default, since without one you cannot use setContents.
- */
- generateDefaultKey() {
- this._default = this.newDefaultKeyBundle();
- },
-
- /**
- * Return true if keys are already present for each of the given
- * collections.
- */
- hasKeysFor(collections) {
- // We can't use filter() here because sometimes collections is an iterator.
- for (let collection of collections) {
- if (!this._collections[collection]) {
- return false;
- }
- }
- return true;
- },
-
- /**
- * Return a new CollectionKeyManager that has keys for each of the
- * given collections (creating new ones for collections where we
- * don't already have keys).
- */
- ensureKeysFor(collections) {
- const newKeys = Object.assign({}, this._collections);
- for (let c of collections) {
- if (newKeys[c]) {
- continue; // don't replace existing keys
- }
-
- const b = new BulkKeyBundle(c);
- b.generateRandom();
- newKeys[c] = b;
- }
- return new CollectionKeyManager(this.lastModified, this._default, newKeys);
- },
-
// Take the fetched info/collections WBO, checking the change
// time of the crypto collection.
updateNeeded: function(info_collections) {
@@ -487,6 +417,9 @@ CollectionKeyManager.prototype = {
//
setContents: function setContents(payload, modified) {
+ if (!modified)
+ throw "No modified time provided to setContents.";
+
let self = this;
this._log.info("Setting collection keys contents. Our last modified: " +
@@ -516,7 +449,9 @@ CollectionKeyManager.prototype = {
if (v) {
let keyObj = new BulkKeyBundle(k);
keyObj.keyPairB64 = v;
- newCollections[k] = keyObj;
+ if (keyObj) {
+ newCollections[k] = keyObj;
+ }
}
}
}
@@ -527,11 +462,8 @@ CollectionKeyManager.prototype = {
let sameColls = collComparison.same;
if (sameDefault && sameColls) {
- self._log.info("New keys are the same as our old keys!");
- if (modified) {
- self._log.info("Bumped local modified time.");
- self.lastModified = modified;
- }
+ self._log.info("New keys are the same as our old keys! Bumped local modified time.");
+ self.lastModified = modified;
return false;
}
@@ -543,10 +475,8 @@ CollectionKeyManager.prototype = {
this._collections = newCollections;
// Always trust the server.
- if (modified) {
- self._log.info("Bumping last modified to " + modified);
- self.lastModified = modified;
- }
+ self._log.info("Bumping last modified to " + modified);
+ self.lastModified = modified;
return sameDefault ? collComparison.changed : true;
},
@@ -594,12 +524,6 @@ this.Collection = function Collection(uri, recordObj, service) {
this._older = 0;
this._newer = 0;
this._data = [];
- // optional members used by batch upload operations.
- this._batch = null;
- this._commit = false;
- // Used for batch download operations -- note that this is explicitly an
- // opaque value and not (necessarily) a number.
- this._offset = null;
}
Collection.prototype = {
__proto__: Resource.prototype,
@@ -623,12 +547,6 @@ Collection.prototype = {
args.push("ids=" + this.ids);
if (this.limit > 0 && this.limit != Infinity)
args.push("limit=" + this.limit);
- if (this._batch)
- args.push("batch=" + encodeURIComponent(this._batch));
- if (this._commit)
- args.push("commit=true");
- if (this._offset)
- args.push("offset=" + encodeURIComponent(this._offset));
this.uri.query = (args.length > 0)? '?' + args.join('&') : '';
},
@@ -641,14 +559,14 @@ Collection.prototype = {
},
// Apply the action to a certain set of ids
- get ids() { return this._ids; },
+ get ids() this._ids,
set ids(value) {
this._ids = value;
this._rebuildURL();
},
// Limit how many records to get
- get limit() { return this._limit; },
+ get limit() this._limit,
set limit(value) {
this._limit = value;
this._rebuildURL();
@@ -678,100 +596,12 @@ Collection.prototype = {
this._rebuildURL();
},
- get offset() { return this._offset; },
- set offset(value) {
- this._offset = value;
- this._rebuildURL();
- },
-
- // Set information about the batch for this request.
- get batch() { return this._batch; },
- set batch(value) {
- this._batch = value;
- this._rebuildURL();
- },
-
- get commit() { return this._commit; },
- set commit(value) {
- this._commit = value && true;
- this._rebuildURL();
+ pushData: function Coll_pushData(data) {
+ this._data.push(data);
},
- // Similar to get(), but will page through the items `batchSize` at a time,
- // deferring calling the record handler until we've gotten them all.
- //
- // Returns the last response processed, and doesn't run the record handler
- // on any items if a non-success status is received while downloading the
- // records (or if a network error occurs).
- getBatched(batchSize = DEFAULT_DOWNLOAD_BATCH_SIZE) {
- let totalLimit = Number(this.limit) || Infinity;
- if (batchSize <= 0 || batchSize >= totalLimit) {
- // Invalid batch sizes should arguably be an error, but they're easy to handle
- return this.get();
- }
-
- if (!this.full) {
- throw new Error("getBatched is unimplemented for guid-only GETs");
- }
-
- // _onComplete and _onProgress are reset after each `get` by AsyncResource.
- // We overwrite _onRecord to something that stores the data in an array
- // until the end.
- let { _onComplete, _onProgress, _onRecord } = this;
- let recordBuffer = [];
- let resp;
- try {
- this._onRecord = r => recordBuffer.push(r);
- let lastModifiedTime;
- this.limit = batchSize;
-
- do {
- this._onProgress = _onProgress;
- this._onComplete = _onComplete;
- if (batchSize + recordBuffer.length > totalLimit) {
- this.limit = totalLimit - recordBuffer.length;
- }
- this._log.trace("Performing batched GET", { limit: this.limit, offset: this.offset });
- // Actually perform the request
- resp = this.get();
- if (!resp.success) {
- break;
- }
-
- // Initialize last modified, or check that something broken isn't happening.
- let lastModified = resp.headers["x-last-modified"];
- if (!lastModifiedTime) {
- lastModifiedTime = lastModified;
- this.setHeader("X-If-Unmodified-Since", lastModified);
- } else if (lastModified != lastModifiedTime) {
- // Should be impossible -- We'd get a 412 in this case.
- throw new Error("X-Last-Modified changed in the middle of a download batch! " +
- `${lastModified} => ${lastModifiedTime}`)
- }
-
- // If this is missing, we're finished.
- this.offset = resp.headers["x-weave-next-offset"];
- } while (this.offset && totalLimit > recordBuffer.length);
- } finally {
- // Ensure we undo any temporary state so that subsequent calls to get()
- // or getBatched() work properly. We do this before calling the record
- // handler so that we can more convincingly pretend to be a normal get()
- // call. Note: we're resetting these to the values they had before this
- // function was called.
- this._onRecord = _onRecord;
- this._limit = totalLimit;
- this._offset = null;
- delete this._headers["x-if-unmodified-since"];
- this._rebuildURL();
- }
- if (resp.success && Async.checkAppReady()) {
- // call the original _onRecord (e.g. the user supplied record handler)
- // for each record we've stored
- for (let record of recordBuffer) {
- this._onRecord(record);
- }
- }
- return resp;
+ clearRecords: function Coll_clearRecords() {
+ this._data = [];
},
set recordHandler(onRecord) {
@@ -781,8 +611,6 @@ Collection.prototype = {
// Switch to newline separated records for incremental parsing
coll.setHeader("Accept", "application/newlines");
- this._onRecord = onRecord;
-
this._onProgress = function() {
let newline;
while ((newline = this._data.indexOf("\n")) > 0) {
@@ -793,247 +621,8 @@ Collection.prototype = {
// Deserialize a record from json and give it to the callback
let record = new coll._recordObj();
record.deserialize(json);
- coll._onRecord(record);
+ onRecord(record);
}
};
},
-
- // This object only supports posting via the postQueue object.
- post() {
- throw new Error("Don't directly post to a collection - use newPostQueue instead");
- },
-
- newPostQueue(log, timestamp, postCallback) {
- let poster = (data, headers, batch, commit) => {
- this.batch = batch;
- this.commit = commit;
- for (let [header, value] of headers) {
- this.setHeader(header, value);
- }
- return Resource.prototype.post.call(this, data);
- }
- let getConfig = (name, defaultVal) => {
- if (this._service.serverConfiguration && this._service.serverConfiguration.hasOwnProperty(name)) {
- return this._service.serverConfiguration[name];
- }
- return defaultVal;
- }
-
- let config = {
- max_post_bytes: getConfig("max_post_bytes", MAX_UPLOAD_BYTES),
- max_post_records: getConfig("max_post_records", MAX_UPLOAD_RECORDS),
-
- max_batch_bytes: getConfig("max_total_bytes", Infinity),
- max_batch_records: getConfig("max_total_records", Infinity),
- }
-
- // Handle config edge cases
- if (config.max_post_records <= 0) { config.max_post_records = MAX_UPLOAD_RECORDS; }
- if (config.max_batch_records <= 0) { config.max_batch_records = Infinity; }
- if (config.max_post_bytes <= 0) { config.max_post_bytes = MAX_UPLOAD_BYTES; }
- if (config.max_batch_bytes <= 0) { config.max_batch_bytes = Infinity; }
-
- // Max size of BSO payload is 256k. This assumes at most 4k of overhead,
- // which sounds like plenty. If the server says it can't handle this, we
- // might have valid records we can't sync, so we give up on syncing.
- let requiredMax = 260 * 1024;
- if (config.max_post_bytes < requiredMax) {
- this._log.error("Server configuration max_post_bytes is too low", config);
- throw new Error("Server configuration max_post_bytes is too low");
- }
-
- return new PostQueue(poster, timestamp, config, log, postCallback);
- },
};
-
-/* A helper to manage the posting of records while respecting the various
- size limits.
-
- This supports the concept of a server-side "batch". The general idea is:
- * We queue as many records as allowed in memory, then make a single POST.
- * This first POST (optionally) gives us a batch ID, which we use for
- all subsequent posts, until...
- * At some point we hit a batch-maximum, and jump through a few hoops to
- commit the current batch (ie, all previous POSTs) and start a new one.
- * Eventually commit the final batch.
-
- In most cases we expect there to be exactly 1 batch consisting of possibly
- multiple POSTs.
-*/
-function PostQueue(poster, timestamp, config, log, postCallback) {
- // The "post" function we should use when it comes time to do the post.
- this.poster = poster;
- this.log = log;
-
- // The config we use. We expect it to have fields "max_post_records",
- // "max_batch_records", "max_post_bytes", and "max_batch_bytes"
- this.config = config;
-
- // The callback we make with the response when we do get around to making the
- // post (which could be during any of the enqueue() calls or the final flush())
- // This callback may be called multiple times and must not add new items to
- // the queue.
- // The second argument passed to this callback is a boolean value that is true
- // if we're in the middle of a batch, and false if either the batch is
- // complete, or it's a post to a server that does not understand batching.
- this.postCallback = postCallback;
-
- // The string where we are capturing the stringified version of the records
- // queued so far. It will always be invalid JSON as it is always missing the
- // closing bracket.
- this.queued = "";
-
- // The number of records we've queued so far but are yet to POST.
- this.numQueued = 0;
-
- // The number of records/bytes we've processed in previous POSTs for our
- // current batch. Does *not* include records currently queued for the next POST.
- this.numAlreadyBatched = 0;
- this.bytesAlreadyBatched = 0;
-
- // The ID of our current batch. Can be undefined (meaning we are yet to make
- // the first post of a patch, so don't know if we have a batch), null (meaning
- // we've made the first post but the server response indicated no batching
- // semantics), otherwise we have made the first post and it holds the batch ID
- // returned from the server.
- this.batchID = undefined;
-
- // Time used for X-If-Unmodified-Since -- should be the timestamp from the last GET.
- this.lastModified = timestamp;
-}
-
-PostQueue.prototype = {
- enqueue(record) {
- // We want to ensure the record has a .toJSON() method defined - even
- // though JSON.stringify() would implicitly call it, the stringify might
- // still work even if it isn't defined, which isn't what we want.
- let jsonRepr = record.toJSON();
- if (!jsonRepr) {
- throw new Error("You must only call this with objects that explicitly support JSON");
- }
- let bytes = JSON.stringify(jsonRepr);
-
- // Do a flush if we can't add this record without exceeding our single-request
- // limits, or without exceeding the total limit for a single batch.
- let newLength = this.queued.length + bytes.length + 2; // extras for leading "[" / "," and trailing "]"
-
- let maxAllowedBytes = Math.min(256 * 1024, this.config.max_post_bytes);
-
- let postSizeExceeded = this.numQueued >= this.config.max_post_records ||
- newLength >= maxAllowedBytes;
-
- let batchSizeExceeded = (this.numQueued + this.numAlreadyBatched) >= this.config.max_batch_records ||
- (newLength + this.bytesAlreadyBatched) >= this.config.max_batch_bytes;
-
- let singleRecordTooBig = bytes.length + 2 > maxAllowedBytes;
-
- if (postSizeExceeded || batchSizeExceeded) {
- this.log.trace(`PostQueue flushing due to postSizeExceeded=${postSizeExceeded}, batchSizeExceeded=${batchSizeExceeded}` +
- `, max_batch_bytes: ${this.config.max_batch_bytes}, max_post_bytes: ${this.config.max_post_bytes}`);
-
- if (singleRecordTooBig) {
- return { enqueued: false, error: new Error("Single record too large to submit to server") };
- }
-
- // We need to write the queue out before handling this one, but we only
- // commit the batch (and thus start a new one) if the batch is full.
- // Note that if a single record is too big for the batch or post, then
- // the batch may be empty, and so we don't flush in that case.
- if (this.numQueued) {
- this.flush(batchSizeExceeded || singleRecordTooBig);
- }
- }
- // Either a ',' or a '[' depending on whether this is the first record.
- this.queued += this.numQueued ? "," : "[";
- this.queued += bytes;
- this.numQueued++;
- return { enqueued: true };
- },
-
- flush(finalBatchPost) {
- if (!this.queued) {
- // nothing queued - we can't be in a batch, and something has gone very
- // bad if we think we are.
- if (this.batchID) {
- throw new Error(`Flush called when no queued records but we are in a batch ${this.batchID}`);
- }
- return;
- }
- // the batch query-param and headers we'll send.
- let batch;
- let headers = [];
- if (this.batchID === undefined) {
- // First commit in a (possible) batch.
- batch = "true";
- } else if (this.batchID) {
- // We have an existing batch.
- batch = this.batchID;
- } else {
- // Not the first post and we know we have no batch semantics.
- batch = null;
- }
-
- headers.push(["x-if-unmodified-since", this.lastModified]);
-
- this.log.info(`Posting ${this.numQueued} records of ${this.queued.length+1} bytes with batch=${batch}`);
- let queued = this.queued + "]";
- if (finalBatchPost) {
- this.bytesAlreadyBatched = 0;
- this.numAlreadyBatched = 0;
- } else {
- this.bytesAlreadyBatched += queued.length;
- this.numAlreadyBatched += this.numQueued;
- }
- this.queued = "";
- this.numQueued = 0;
- let response = this.poster(queued, headers, batch, !!(finalBatchPost && this.batchID !== null));
-
- if (!response.success) {
- this.log.trace("Server error response during a batch", response);
- // not clear what we should do here - we expect the consumer of this to
- // abort by throwing in the postCallback below.
- return this.postCallback(response, !finalBatchPost);
- }
-
- if (finalBatchPost) {
- this.log.trace("Committed batch", this.batchID);
- this.batchID = undefined; // we are now in "first post for the batch" state.
- this.lastModified = response.headers["x-last-modified"];
- return this.postCallback(response, false);
- }
-
- if (response.status != 202) {
- if (this.batchID) {
- throw new Error("Server responded non-202 success code while a batch was in progress");
- }
- this.batchID = null; // no batch semantics are in place.
- this.lastModified = response.headers["x-last-modified"];
- return this.postCallback(response, false);
- }
-
- // this response is saying the server has batch semantics - we should
- // always have a batch ID in the response.
- let responseBatchID = response.obj.batch;
- this.log.trace("Server responsed 202 with batch", responseBatchID);
- if (!responseBatchID) {
- this.log.error("Invalid server response: 202 without a batch ID", response);
- throw new Error("Invalid server response: 202 without a batch ID");
- }
-
- if (this.batchID === undefined) {
- this.batchID = responseBatchID;
- if (!this.lastModified) {
- this.lastModified = response.headers["x-last-modified"];
- if (!this.lastModified) {
- throw new Error("Batch response without x-last-modified");
- }
- }
- }
-
- if (this.batchID != responseBatchID) {
- throw new Error(`Invalid client/server batch state - client has ${this.batchID}, server has ${responseBatchID}`);
- }
-
- this.postCallback(response, true);
- },
-}
diff --git a/services/sync/modules/resource.js b/services/sync/modules/resource.js
index bf7066b9f..1c2a67b90 100644
--- a/services/sync/modules/resource.js
+++ b/services/sync/modules/resource.js
@@ -7,13 +7,12 @@ this.EXPORTED_SYMBOLS = [
"Resource"
];
-var Cc = Components.classes;
-var Ci = Components.interfaces;
-var Cr = Components.results;
-var Cu = Components.utils;
+const Cc = Components.classes;
+const Ci = Components.interfaces;
+const Cr = Components.results;
+const Cu = Components.utils;
Cu.import("resource://gre/modules/Preferences.jsm");
-Cu.import("resource://gre/modules/NetUtil.jsm");
Cu.import("resource://services-common/async.js");
Cu.import("resource://gre/modules/Log.jsm");
Cu.import("resource://services-common/observers.js");
@@ -74,6 +73,20 @@ AsyncResource.prototype = {
*/
authenticator: null,
+ // The string to use as the base User-Agent in Sync requests.
+ // These strings will look something like
+ //
+ // Firefox/4.0 FxSync/1.8.0.20100101.mobile
+ //
+ // or
+ //
+ // Firefox Aurora/5.0a1 FxSync/1.9.0.20110409.desktop
+ //
+ _userAgent:
+ Services.appinfo.name + "/" + Services.appinfo.version + // Product.
+ " FxSync/" + WEAVE_VERSION + "." + // Sync.
+ Services.appinfo.appBuildID + ".", // Build.
+
// Wait 5 minutes before killing a request.
ABORT_TIMEOUT: 300000,
@@ -121,9 +134,7 @@ AsyncResource.prototype = {
//
// Get and set the data encapulated in the resource.
_data: null,
- get data() {
- return this._data;
- },
+ get data() this._data,
set data(value) {
this._data = value;
},
@@ -135,9 +146,16 @@ AsyncResource.prototype = {
// to obtain a request channel.
//
_createRequest: function Res__createRequest(method) {
- let channel = NetUtil.newChannel({uri: this.spec, loadUsingSystemPrincipal: true})
- .QueryInterface(Ci.nsIRequest)
- .QueryInterface(Ci.nsIHttpChannel);
+ let channel = Services.io.newChannel2(this.spec,
+ null,
+ null,
+ null, // aLoadingNode
+ Services.scriptSecurityManager.getSystemPrincipal(),
+ null, // aTriggeringPrincipal
+ Ci.nsILoadInfo.SEC_NORMAL,
+ Ci.nsIContentPolicy.TYPE_OTHER)
+ .QueryInterface(Ci.nsIRequest)
+ .QueryInterface(Ci.nsIHttpChannel);
channel.loadFlags |= DEFAULT_LOAD_FLAGS;
@@ -147,7 +165,8 @@ AsyncResource.prototype = {
// Compose a UA string fragment from the various available identifiers.
if (Svc.Prefs.get("sendVersionInfo", true)) {
- channel.setRequestHeader("user-agent", Utils.userAgent, false);
+ let ua = this._userAgent + Svc.Prefs.get("client.type", "desktop");
+ channel.setRequestHeader("user-agent", ua, false);
}
let headers = this.headers;
@@ -155,7 +174,7 @@ AsyncResource.prototype = {
if (this.authenticator) {
let result = this.authenticator(this, method);
if (result && result.headers) {
- for (let [k, v] of Object.entries(result.headers)) {
+ for (let [k, v] in Iterator(result.headers)) {
headers[k.toLowerCase()] = v;
}
}
@@ -163,7 +182,7 @@ AsyncResource.prototype = {
this._log.debug("No authenticator found.");
}
- for (let [key, value] of Object.entries(headers)) {
+ for (let [key, value] in Iterator(headers)) {
if (key == 'authorization')
this._log.trace("HTTP Header " + key + ": ***** (suppressed)");
else
@@ -209,10 +228,10 @@ AsyncResource.prototype = {
this._log, this.ABORT_TIMEOUT);
channel.requestMethod = action;
try {
- channel.asyncOpen2(listener);
+ channel.asyncOpen(listener, null);
} catch (ex) {
- // asyncOpen2 can throw in a bunch of cases -- e.g., a forbidden port.
- this._log.warn("Caught an error in asyncOpen2", ex);
+ // asyncOpen can throw in a bunch of cases -- e.g., a forbidden port.
+ this._log.warn("Caught an error in asyncOpen: " + CommonUtils.exceptionStr(ex));
CommonUtils.nextTick(callback.bind(this, ex));
}
},
@@ -259,7 +278,9 @@ AsyncResource.prototype = {
} catch(ex) {
// Got a response, but an exception occurred during processing.
// This shouldn't occur.
- this._log.warn("Caught unexpected exception in _oncomplete", ex);
+ this._log.warn("Caught unexpected exception " + CommonUtils.exceptionStr(ex) +
+ " in _onComplete.");
+ this._log.debug(CommonUtils.stackTrace(ex));
}
// Process headers. They can be empty, or the call can otherwise fail, so
@@ -297,18 +318,16 @@ AsyncResource.prototype = {
contentLength + ".");
}
} catch (ex) {
- this._log.debug("Caught exception visiting headers in _onComplete", ex);
+ this._log.debug("Caught exception " + CommonUtils.exceptionStr(ex) +
+ " visiting headers in _onComplete.");
+ this._log.debug(CommonUtils.stackTrace(ex));
}
let ret = new String(data);
- ret.url = channel.URI.spec;
ret.status = status;
ret.success = success;
ret.headers = headers;
- if (!success) {
- this._log.warn(`${action} request to ${ret.url} failed with status ${status}`);
- }
// Make a lazy getter to convert the json response into an object.
// Note that this can cause a parse error to be thrown far away from the
// actual fetch, so be warned!
@@ -316,7 +335,7 @@ AsyncResource.prototype = {
try {
return JSON.parse(ret);
} catch (ex) {
- this._log.warn("Got exception parsing response body", ex);
+ this._log.warn("Got exception parsing response body: \"" + CommonUtils.exceptionStr(ex));
// Stringify to avoid possibly printing non-printable characters.
this._log.debug("Parse fail: Response body starts: \"" +
JSON.stringify((ret + "").slice(0, 100)) +
@@ -384,12 +403,7 @@ Resource.prototype = {
try {
this._doRequest(action, data, callback);
return Async.waitForSyncCallback(cb);
- } catch (ex) {
- if (Async.isShutdownException(ex)) {
- throw ex;
- }
- this._log.warn("${action} request to ${url} failed: ${ex}",
- { action, url: this.uri.spec, ex });
+ } catch(ex) {
// Combine the channel stack with this request stack. Need to create
// a new error object for that.
let error = Error(ex.message);
@@ -527,7 +541,7 @@ ChannelListener.prototype = {
siStream = Cc["@mozilla.org/scriptableinputstream;1"].createInstance(Ci.nsIScriptableInputStream);
siStream.init(stream);
} catch (ex) {
- this._log.warn("Exception creating nsIScriptableInputStream", ex);
+ this._log.warn("Exception creating nsIScriptableInputStream." + CommonUtils.exceptionStr(ex));
this._log.debug("Parameters: " + req.URI.spec + ", " + stream + ", " + off + ", " + count);
// Cannot proceed, so rethrow and allow the channel to cancel itself.
throw ex;
@@ -543,11 +557,9 @@ ChannelListener.prototype = {
try {
this._onProgress();
} catch (ex) {
- if (Async.isShutdownException(ex)) {
- throw ex;
- }
this._log.warn("Got exception calling onProgress handler during fetch of "
- + req.URI.spec, ex);
+ + req.URI.spec);
+ this._log.debug(CommonUtils.exceptionStr(ex));
this._log.trace("Rethrowing; expect a failure code from the HTTP channel.");
throw ex;
}
@@ -562,7 +574,7 @@ ChannelListener.prototype = {
try {
CommonUtils.namedTimer(this.abortRequest, this._timeout, this, "abortTimer");
} catch (ex) {
- this._log.warn("Got exception extending abort timer", ex);
+ this._log.warn("Got exception extending abort timer: " + CommonUtils.exceptionStr(ex));
}
},
@@ -656,14 +668,14 @@ ChannelNotificationListener.prototype = {
}
}
} catch (ex) {
- this._log.error("Error copying headers", ex);
+ this._log.error("Error copying headers: " + CommonUtils.exceptionStr(ex));
}
// We let all redirects proceed.
try {
callback.onRedirectVerifyCallback(Cr.NS_OK);
} catch (ex) {
- this._log.error("onRedirectVerifyCallback threw!", ex);
+ this._log.error("onRedirectVerifyCallback threw!" + CommonUtils.exceptionStr(ex));
}
}
};
diff --git a/services/sync/modules/rest.js b/services/sync/modules/rest.js
index 94c096dba..34382eed5 100644
--- a/services/sync/modules/rest.js
+++ b/services/sync/modules/rest.js
@@ -2,7 +2,7 @@
* License, v. 2.0. If a copy of the MPL was not distributed with this
* file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-var {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
+const {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
Cu.import("resource://gre/modules/Log.jsm");
Cu.import("resource://services-common/rest.js");
@@ -28,6 +28,21 @@ SyncStorageRequest.prototype = {
_logName: "Sync.StorageRequest",
/**
+ * The string to use as the base User-Agent in Sync requests.
+ * These strings will look something like
+ *
+ * Firefox/4.0 FxSync/1.8.0.20100101.mobile
+ *
+ * or
+ *
+ * Firefox Aurora/5.0a1 FxSync/1.9.0.20110409.desktop
+ */
+ userAgent:
+ Services.appinfo.name + "/" + Services.appinfo.version + // Product.
+ " FxSync/" + WEAVE_VERSION + "." + // Sync.
+ Services.appinfo.appBuildID + ".", // Build.
+
+ /**
* Wait 5 minutes before killing a request.
*/
timeout: STORAGE_REQUEST_TIMEOUT,
@@ -35,7 +50,8 @@ SyncStorageRequest.prototype = {
dispatch: function dispatch(method, data, onComplete, onProgress) {
// Compose a UA string fragment from the various available identifiers.
if (Svc.Prefs.get("sendVersionInfo", true)) {
- this.setHeader("user-agent", Utils.userAgent);
+ let ua = this.userAgent + Svc.Prefs.get("client.type", "desktop");
+ this.setHeader("user-agent", ua);
}
if (this.authenticator) {
diff --git a/services/sync/modules/service.js b/services/sync/modules/service.js
index 32e047f53..637760b8f 100644
--- a/services/sync/modules/service.js
+++ b/services/sync/modules/service.js
@@ -4,10 +4,10 @@
this.EXPORTED_SYMBOLS = ["Service"];
-var Cc = Components.classes;
-var Ci = Components.interfaces;
-var Cr = Components.results;
-var Cu = Components.utils;
+const Cc = Components.classes;
+const Ci = Components.interfaces;
+const Cr = Components.results;
+const Cu = Components.utils;
// How long before refreshing the cluster
const CLUSTER_BACKOFF = 5 * 60 * 1000; // 5 minutes
@@ -21,6 +21,7 @@ const KEYS_WBO = "keys";
Cu.import("resource://gre/modules/Preferences.jsm");
Cu.import("resource://gre/modules/XPCOMUtils.jsm");
Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://services-common/utils.js");
Cu.import("resource://services-sync/constants.js");
Cu.import("resource://services-sync/engines.js");
Cu.import("resource://services-sync/engines/clients.js");
@@ -32,7 +33,6 @@ Cu.import("resource://services-sync/rest.js");
Cu.import("resource://services-sync/stages/enginesync.js");
Cu.import("resource://services-sync/stages/declined.js");
Cu.import("resource://services-sync/status.js");
-Cu.import("resource://services-sync/telemetry.js");
Cu.import("resource://services-sync/userapi.js");
Cu.import("resource://services-sync/util.js");
@@ -51,6 +51,15 @@ const STORAGE_INFO_TYPES = [INFO_COLLECTIONS,
INFO_COLLECTION_COUNTS,
INFO_QUOTA];
+// A structure mapping a (boolean) telemetry probe name to a preference name.
+// The probe will record true if the pref is modified, false otherwise.
+const TELEMETRY_CUSTOM_SERVER_PREFS = {
+ WEAVE_CUSTOM_LEGACY_SERVER_CONFIGURATION: "services.sync.serverURL",
+ WEAVE_CUSTOM_FXA_SERVER_CONFIGURATION: "identity.fxaccounts.auth.uri",
+ WEAVE_CUSTOM_TOKEN_SERVER_CONFIGURATION: "services.sync.tokenServerURI",
+};
+
+
function Sync11Service() {
this._notify = Utils.notify("weave:service:");
}
@@ -64,13 +73,8 @@ Sync11Service.prototype = {
storageURL: null,
metaURL: null,
cryptoKeyURL: null,
- // The cluster URL comes via the ClusterManager object, which in the FxA
- // world is ebbedded in the token returned from the token server.
- _clusterURL: null,
- get serverURL() {
- return Svc.Prefs.get("serverURL");
- },
+ get serverURL() Svc.Prefs.get("serverURL"),
set serverURL(value) {
if (!value.endsWith("/")) {
value += "/";
@@ -80,20 +84,14 @@ Sync11Service.prototype = {
if (value == this.serverURL)
return;
+ // A new server most likely uses a different cluster, so clear that
Svc.Prefs.set("serverURL", value);
-
- // A new server most likely uses a different cluster, so clear that.
- this._clusterURL = null;
+ Svc.Prefs.reset("clusterURL");
},
- get clusterURL() {
- return this._clusterURL || "";
- },
+ get clusterURL() Svc.Prefs.get("clusterURL", ""),
set clusterURL(value) {
- if (value != null && typeof value != "string") {
- throw new Error("cluster must be a string, got " + (typeof value));
- }
- this._clusterURL = value;
+ Svc.Prefs.set("clusterURL", value);
this._updateCachedURLs();
},
@@ -171,16 +169,8 @@ Sync11Service.prototype = {
_updateCachedURLs: function _updateCachedURLs() {
// Nothing to cache yet if we don't have the building blocks
- if (!this.clusterURL || !this.identity.username) {
- // Also reset all other URLs used by Sync to ensure we aren't accidentally
- // using one cached earlier - if there's no cluster URL any cached ones
- // are invalid.
- this.infoURL = undefined;
- this.storageURL = undefined;
- this.metaURL = undefined;
- this.cryptoKeysURL = undefined;
+ if (!this.clusterURL || !this.identity.username)
return;
- }
this._log.debug("Caching URLs under storage user base: " + this.userBaseURL);
@@ -315,6 +305,21 @@ Sync11Service.prototype = {
return false;
},
+ // The global "enabled" state comes from prefs, and will be set to false
+ // whenever the UI that exposes what to sync finds all Sync engines disabled.
+ get enabled() {
+ return Svc.Prefs.get("enabled");
+ },
+ set enabled(val) {
+ // There's no real reason to impose this other than to catch someone doing
+ // something we don't expect with bad consequences - all setting of this
+ // pref are in the UI code and external to this module.
+ if (val) {
+ throw new Error("Only disabling via this setter is supported");
+ }
+ Svc.Prefs.set("enabled", val);
+ },
+
/**
* Prepare to initialize the rest of Weave after waiting a little bit
*/
@@ -344,8 +349,6 @@ Sync11Service.prototype = {
this._clusterManager = this.identity.createClusterManager(this);
this.recordManager = new RecordManager(this);
- this.enabled = true;
-
this._registerEngines();
let ua = Cc["@mozilla.org/network/protocol;1?name=http"].
@@ -359,7 +362,6 @@ Sync11Service.prototype = {
}
Svc.Obs.add("weave:service:setup-complete", this);
- Svc.Obs.add("sync:collection_changed", this); // Pulled from FxAccountsCommon
Svc.Prefs.observe("engine.", this);
this.scheduler = new SyncScheduler(this);
@@ -375,6 +377,12 @@ Sync11Service.prototype = {
Svc.Obs.notify("weave:engine:start-tracking");
}
+ // Telemetry probes to indicate if the user is using custom servers.
+ for (let [probeName, prefName] of Iterator(TELEMETRY_CUSTOM_SERVER_PREFS)) {
+ let isCustomized = Services.prefs.prefHasUserValue(prefName);
+ Services.telemetry.getHistogramById(probeName).add(isCustomized);
+ }
+
// Send an event now that Weave service is ready. We don't do this
// synchronously so that observers can import this module before
// registering an observer.
@@ -424,7 +432,7 @@ Sync11Service.prototype = {
// Map each old pref to the current pref branch
let oldPref = new Preferences(oldPrefBranch);
- for (let pref of oldPrefNames)
+ for each (let pref in oldPrefNames)
Svc.Prefs.set(pref, oldPref.get(pref));
// Remove all the old prefs and remember that we've migrated
@@ -472,7 +480,8 @@ Sync11Service.prototype = {
this.engineManager.register(ns[engineName]);
} catch (ex) {
- this._log.warn("Could not register engine " + name, ex);
+ this._log.warn("Could not register engine " + name + ": " +
+ CommonUtils.exceptionStr(ex));
}
}
@@ -486,13 +495,6 @@ Sync11Service.prototype = {
observe: function observe(subject, topic, data) {
switch (topic) {
- // Ideally this observer should be in the SyncScheduler, but it would require
- // some work to know about the sync specific engines. We should move this there once it does.
- case "sync:collection_changed":
- if (data.includes("clients")) {
- this.sync([]); // [] = clients collection only
- }
- break;
case "weave:service:setup-complete":
let status = this._checkSetup();
if (status != STATUS_DISABLED && status != CLIENT_NOT_CONFIGURED)
@@ -557,8 +559,7 @@ Sync11Service.prototype = {
// Always check for errors; this is also where we look for X-Weave-Alert.
this.errorHandler.checkServerError(info);
if (!info.success) {
- this._log.error("Aborting sync: failed to get collections.")
- throw info;
+ throw "Aborting sync: failed to get collections.";
}
return info;
},
@@ -675,13 +676,21 @@ Sync11Service.prototype = {
} catch (ex) {
// This means no keys are present, or there's a network error.
- this._log.debug("Failed to fetch and verify keys", ex);
+ this._log.debug("Failed to fetch and verify keys: "
+ + Utils.exceptionStr(ex));
this.errorHandler.checkServerError(ex);
return false;
}
},
verifyLogin: function verifyLogin(allow40XRecovery = true) {
+ // If the identity isn't ready it might not know the username...
+ if (!this.identity.readyToAuthenticate) {
+ this._log.info("Not ready to authenticate in verifyLogin.");
+ this.status.login = LOGIN_FAILED_NOT_READY;
+ return false;
+ }
+
if (!this.identity.username) {
this._log.warn("No username in verifyLogin.");
this.status.login = LOGIN_FAILED_NO_USERNAME;
@@ -753,12 +762,8 @@ Sync11Service.prototype = {
return this.verifyLogin(false);
}
- // We must have the right cluster, but the server doesn't expect us.
- // The implications of this depend on the identity being used - for
- // the legacy identity, it's an authoritatively "incorrect password",
- // (ie, LOGIN_FAILED_LOGIN_REJECTED) but for FxA it probably means
- // "transient error fetching auth token".
- this.status.login = this.identity.loginStatusFromVerification404();
+ // We must have the right cluster, but the server doesn't expect us
+ this.status.login = LOGIN_FAILED_LOGIN_REJECTED;
return false;
default:
@@ -769,7 +774,7 @@ Sync11Service.prototype = {
}
} catch (ex) {
// Must have failed on some network issue
- this._log.debug("verifyLogin failed", ex);
+ this._log.debug("verifyLogin failed: " + Utils.exceptionStr(ex));
this.status.login = LOGIN_FAILED_NETWORK_ERROR;
this.errorHandler.checkServerError(ex);
return false;
@@ -842,7 +847,8 @@ Sync11Service.prototype = {
try {
cb.wait();
} catch (ex) {
- this._log.debug("Password change failed", ex);
+ this._log.debug("Password change failed: " +
+ CommonUtils.exceptionStr(ex));
return false;
}
@@ -884,11 +890,12 @@ Sync11Service.prototype = {
// Deletion doesn't make sense if we aren't set up yet!
if (this.clusterURL != "") {
// Clear client-specific data from the server, including disabled engines.
- for (let engine of [this.clientsEngine].concat(this.engineManager.getAll())) {
+ for each (let engine in [this.clientsEngine].concat(this.engineManager.getAll())) {
try {
engine.removeClientData();
} catch(ex) {
- this._log.warn(`Deleting client data for ${engine.name} failed`, ex);
+ this._log.warn("Deleting client data for " + engine.name + " failed:"
+ + Utils.exceptionStr(ex));
}
}
this._log.debug("Finished deleting client data.");
@@ -914,7 +921,6 @@ Sync11Service.prototype = {
this._ignorePrefObserver = true;
Svc.Prefs.resetBranch("");
this._ignorePrefObserver = false;
- this.clusterURL = null;
Svc.Prefs.set("lastversion", WEAVE_VERSION);
@@ -931,22 +937,25 @@ Sync11Service.prototype = {
return;
}
- try {
- this.identity.finalize();
- // an observer so the FxA migration code can take some action before
- // the new identity is created.
- Svc.Obs.notify("weave:service:start-over:init-identity");
- this.identity.username = "";
- this.status.__authManager = null;
- this.identity = Status._authManager;
- this._clusterManager = this.identity.createClusterManager(this);
- Svc.Obs.notify("weave:service:start-over:finish");
- } catch (err) {
- this._log.error("startOver failed to re-initialize the identity manager: " + err);
- // Still send the observer notification so the current state is
- // reflected in the UI.
- Svc.Obs.notify("weave:service:start-over:finish");
- }
+ this.identity.finalize().then(
+ () => {
+ // an observer so the FxA migration code can take some action before
+ // the new identity is created.
+ Svc.Obs.notify("weave:service:start-over:init-identity");
+ this.identity.username = "";
+ this.status.__authManager = null;
+ this.identity = Status._authManager;
+ this._clusterManager = this.identity.createClusterManager(this);
+ Svc.Obs.notify("weave:service:start-over:finish");
+ }
+ ).then(null,
+ err => {
+ this._log.error("startOver failed to re-initialize the identity manager: " + err);
+ // Still send the observer notification so the current state is
+ // reflected in the UI.
+ Svc.Obs.notify("weave:service:start-over:finish");
+ }
+ );
},
persistLogin: function persistLogin() {
@@ -981,12 +990,8 @@ Sync11Service.prototype = {
}
// Ask the identity manager to explicitly login now.
- this._log.info("Logging in the user.");
let cb = Async.makeSpinningCallback();
- this.identity.ensureLoggedIn().then(
- () => cb(null),
- err => cb(err || "ensureLoggedIn failed")
- );
+ this.identity.ensureLoggedIn().then(cb, cb);
// Just let any errors bubble up - they've more context than we do!
cb.wait();
@@ -997,9 +1002,9 @@ Sync11Service.prototype = {
&& (username || password || passphrase)) {
Svc.Obs.notify("weave:service:setup-complete");
}
+ this._log.info("Logging in the user.");
this._updateCachedURLs();
- this._log.info("User logged in successfully - verifying login.");
if (!this.verifyLogin()) {
// verifyLogin sets the failure states here.
throw "Login failed: " + this.status.login;
@@ -1064,49 +1069,11 @@ Sync11Service.prototype = {
}
},
- // Note: returns false if we failed for a reason other than the server not yet
- // supporting the api.
- _fetchServerConfiguration() {
- if (Svc.Prefs.get("APILevel") >= 2) {
- // This is similar to _fetchInfo, but with different error handling.
- // Only supported by later sync implementations.
-
- let infoURL = this.userBaseURL + "info/configuration";
- this._log.debug("Fetching server configuration", infoURL);
- let configResponse;
- try {
- configResponse = this.resource(infoURL).get();
- } catch (ex) {
- // This is probably a network or similar error.
- this._log.warn("Failed to fetch info/configuration", ex);
- this.errorHandler.checkServerError(ex);
- return false;
- }
-
- if (configResponse.status == 404) {
- // This server doesn't support the URL yet - that's OK.
- this._log.debug("info/configuration returned 404 - using default upload semantics");
- } else if (configResponse.status != 200) {
- this._log.warn(`info/configuration returned ${configResponse.status} - using default configuration`);
- this.errorHandler.checkServerError(configResponse);
- return false;
- } else {
- this.serverConfiguration = configResponse.obj;
- }
- this._log.trace("info/configuration for this server", this.serverConfiguration);
- }
- return true;
- },
-
// Stuff we need to do after login, before we can really do
// anything (e.g. key setup).
_remoteSetup: function _remoteSetup(infoResponse) {
let reset = false;
- if (!this._fetchServerConfiguration()) {
- return false;
- }
-
this._log.debug("Fetching global metadata record");
let meta = this.recordManager.get(this.metaURL);
@@ -1133,7 +1100,7 @@ Sync11Service.prototype = {
return false;
}
- if (this.recordManager.response.status == 404) {
+ if (!this.recordManager.response.success || !newMeta) {
this._log.debug("No meta/global record on the server. Creating one.");
newMeta = new WBORecord("meta", "global");
newMeta.payload.syncID = this.syncID;
@@ -1143,16 +1110,10 @@ Sync11Service.prototype = {
newMeta.isNew = true;
this.recordManager.set(this.metaURL, newMeta);
- let uploadRes = newMeta.upload(this.resource(this.metaURL));
- if (!uploadRes.success) {
+ if (!newMeta.upload(this.resource(this.metaURL)).success) {
this._log.warn("Unable to upload new meta/global. Failing remote setup.");
- this.errorHandler.checkServerError(uploadRes);
return false;
}
- } else if (!newMeta) {
- this._log.warn("Unable to get meta/global. Failing remote setup.");
- this.errorHandler.checkServerError(this.recordManager.response);
- return false;
} else {
// If newMeta, then it stands to reason that meta != null.
newMeta.isNew = meta.isNew;
@@ -1293,9 +1254,13 @@ Sync11Service.prototype = {
return reason;
},
- sync: function sync(engineNamesToSync) {
- let dateStr = Utils.formatTimestamp(new Date());
- this._log.debug("User-Agent: " + Utils.userAgent);
+ sync: function sync() {
+ if (!this.enabled) {
+ this._log.debug("Not syncing as Sync is disabled.");
+ return;
+ }
+ let dateStr = new Date().toLocaleFormat(LOG_DATE_FORMAT);
+ this._log.debug("User-Agent: " + SyncStorageRequest.prototype.userAgent);
this._log.info("Starting sync at " + dateStr);
this._catch(function () {
// Make sure we're logged in.
@@ -1309,14 +1274,14 @@ Sync11Service.prototype = {
else {
this._log.trace("In sync: no need to login.");
}
- return this._lockedSync(engineNamesToSync);
+ return this._lockedSync.apply(this, arguments);
})();
},
/**
* Sync up engines with the server.
*/
- _lockedSync: function _lockedSync(engineNamesToSync) {
+ _lockedSync: function _lockedSync() {
return this._lock("service.js: sync",
this._notify("sync", "", function onNotify() {
@@ -1327,7 +1292,7 @@ Sync11Service.prototype = {
let cb = Async.makeSpinningCallback();
synchronizer.onComplete = cb;
- synchronizer.sync(engineNamesToSync);
+ synchronizer.sync();
// wait() throws if the first argument is truthy, which is exactly what
// we want.
let result = cb.wait();
@@ -1338,31 +1303,27 @@ Sync11Service.prototype = {
// We successfully synchronized.
// Check if the identity wants to pre-fetch a migration sentinel from
// the server.
- // Only supported by Sync server API level 2+
// If we have no clusterURL, we are probably doing a node reassignment
// so don't attempt to get it in that case.
- if (Svc.Prefs.get("APILevel") >= 2 && this.clusterURL) {
- this.identity.prefetchMigrationSentinel(this);
+ //if (this.clusterURL) {
+ // this.identity.prefetchMigrationSentinel(this);
+ //}
+
+ // Now let's update our declined engines.
+ let meta = this.recordManager.get(this.metaURL);
+ if (!meta) {
+ this._log.warn("No meta/global; can't update declined state.");
+ return;
}
- // Now let's update our declined engines (but only if we have a metaURL;
- // if Sync failed due to no node we will not have one)
- if (this.metaURL) {
- let meta = this.recordManager.get(this.metaURL);
- if (!meta) {
- this._log.warn("No meta/global; can't update declined state.");
- return;
- }
-
- let declinedEngines = new DeclinedEngines(this);
- let didChange = declinedEngines.updateDeclined(meta, this.engineManager);
- if (!didChange) {
- this._log.info("No change to declined engines. Not reuploading meta/global.");
- return;
- }
-
- this.uploadMetaGlobal(meta);
+ let declinedEngines = new DeclinedEngines(this);
+ let didChange = declinedEngines.updateDeclined(meta, this.engineManager);
+ if (!didChange) {
+ this._log.info("No change to declined engines. Not reuploading meta/global.");
+ return;
}
+
+ this.uploadMetaGlobal(meta);
}))();
},
@@ -1536,7 +1497,7 @@ Sync11Service.prototype = {
// Wipe everything we know about except meta because we just uploaded it
let engines = [this.clientsEngine].concat(this.engineManager.getAll());
- let collections = engines.map(engine => engine.name);
+ let collections = [engine.name for each (engine in engines)];
// TODO: there's a bug here. We should be calling resetClient, no?
// Generate, upload, and download new keys. Do this last so we don't wipe
@@ -1555,7 +1516,6 @@ Sync11Service.prototype = {
*/
wipeServer: function wipeServer(collections) {
let response;
- let histogram = Services.telemetry.getHistogramById("WEAVE_WIPE_SERVER_SUCCEEDED");
if (!collections) {
// Strip the trailing slash.
let res = this.resource(this.storageURL.slice(0, -1));
@@ -1563,17 +1523,14 @@ Sync11Service.prototype = {
try {
response = res.delete();
} catch (ex) {
- this._log.debug("Failed to wipe server", ex);
- histogram.add(false);
+ this._log.debug("Failed to wipe server: " + CommonUtils.exceptionStr(ex));
throw ex;
}
if (response.status != 200 && response.status != 404) {
this._log.debug("Aborting wipeServer. Server responded with " +
response.status + " response for " + this.storageURL);
- histogram.add(false);
throw response;
}
- histogram.add(true);
return response.headers["x-weave-timestamp"];
}
@@ -1583,15 +1540,14 @@ Sync11Service.prototype = {
try {
response = this.resource(url).delete();
} catch (ex) {
- this._log.debug("Failed to wipe '" + name + "' collection", ex);
- histogram.add(false);
+ this._log.debug("Failed to wipe '" + name + "' collection: " +
+ Utils.exceptionStr(ex));
throw ex;
}
if (response.status != 200 && response.status != 404) {
this._log.debug("Aborting wipeServer. Server responded with " +
response.status + " response for " + url);
- histogram.add(false);
throw response;
}
@@ -1599,7 +1555,7 @@ Sync11Service.prototype = {
timestamp = response.headers["x-weave-timestamp"];
}
}
- histogram.add(true);
+
return timestamp;
},
@@ -1623,7 +1579,7 @@ Sync11Service.prototype = {
}
// Fully wipe each engine if it's able to decrypt data
- for (let engine of engines) {
+ for each (let engine in engines) {
if (engine.canDecrypt()) {
engine.wipeClient();
}
@@ -1701,7 +1657,7 @@ Sync11Service.prototype = {
}
// Have each engine drop any temporary meta data
- for (let engine of engines) {
+ for each (let engine in engines) {
engine.resetClient();
}
})();
@@ -1731,7 +1687,8 @@ Sync11Service.prototype = {
return this.getStorageRequest(url).get(function onComplete(error) {
// Note: 'this' is the request.
if (error) {
- this._log.debug("Failed to retrieve '" + info_type + "'", error);
+ this._log.debug("Failed to retrieve '" + info_type + "': " +
+ Utils.exceptionStr(error));
return callback(error);
}
if (this.response.status != 200) {
diff --git a/services/sync/modules/stages/cluster.js b/services/sync/modules/stages/cluster.js
index 7665ce825..dd358bf98 100644
--- a/services/sync/modules/stages/cluster.js
+++ b/services/sync/modules/stages/cluster.js
@@ -4,7 +4,7 @@
this.EXPORTED_SYMBOLS = ["ClusterManager"];
-var {utils: Cu} = Components;
+const {utils: Cu} = Components;
Cu.import("resource://gre/modules/Log.jsm");
Cu.import("resource://services-sync/constants.js");
@@ -80,9 +80,6 @@ ClusterManager.prototype = {
return false;
}
- // Convert from the funky "String object with additional properties" that
- // resource.js returns to a plain-old string.
- cluster = cluster.toString();
// Don't update stuff if we already have the right cluster
if (cluster == this.service.clusterURL) {
return false;
@@ -90,6 +87,7 @@ ClusterManager.prototype = {
this._log.debug("Setting cluster to " + cluster);
this.service.clusterURL = cluster;
+ Svc.Prefs.set("lastClusterUpdate", Date.now().toString());
return true;
},
diff --git a/services/sync/modules/stages/declined.js b/services/sync/modules/stages/declined.js
index ff8a14181..b0877e929 100644
--- a/services/sync/modules/stages/declined.js
+++ b/services/sync/modules/stages/declined.js
@@ -11,7 +11,7 @@
this.EXPORTED_SYMBOLS = ["DeclinedEngines"];
-var {utils: Cu} = Components;
+const {utils: Cu} = Components;
Cu.import("resource://services-sync/constants.js");
Cu.import("resource://gre/modules/Log.jsm");
@@ -29,8 +29,8 @@ this.DeclinedEngines = function (service) {
}
this.DeclinedEngines.prototype = {
updateDeclined: function (meta, engineManager=this.service.engineManager) {
- let enabled = new Set(engineManager.getEnabled().map(e => e.name));
- let known = new Set(engineManager.getAll().map(e => e.name));
+ let enabled = new Set([e.name for each (e in engineManager.getEnabled())]);
+ let known = new Set([e.name for each (e in engineManager.getAll())]);
let remoteDeclined = new Set(meta.payload.declined || []);
let localDeclined = new Set(engineManager.getDeclined());
diff --git a/services/sync/modules/stages/enginesync.js b/services/sync/modules/stages/enginesync.js
index a00a2f48b..ed91adddb 100644
--- a/services/sync/modules/stages/enginesync.js
+++ b/services/sync/modules/stages/enginesync.js
@@ -8,16 +8,13 @@
this.EXPORTED_SYMBOLS = ["EngineSynchronizer"];
-var {utils: Cu} = Components;
+const {utils: Cu} = Components;
Cu.import("resource://gre/modules/Log.jsm");
Cu.import("resource://services-sync/constants.js");
Cu.import("resource://services-sync/engines.js");
Cu.import("resource://services-sync/policies.js");
Cu.import("resource://services-sync/util.js");
-Cu.import("resource://services-common/observers.js");
-Cu.import("resource://services-common/async.js");
-Cu.import("resource://gre/modules/Task.jsm");
/**
* Perform synchronization of engines.
@@ -34,7 +31,7 @@ this.EngineSynchronizer = function EngineSynchronizer(service) {
}
EngineSynchronizer.prototype = {
- sync: function sync(engineNamesToSync) {
+ sync: function sync() {
if (!this.onComplete) {
throw new Error("onComplete handler not installed.");
}
@@ -99,9 +96,6 @@ EngineSynchronizer.prototype = {
return;
}
- // We only honor the "hint" of what engines to Sync if this isn't
- // a first sync.
- let allowEnginesHint = false;
// Wipe data in the desired direction if necessary
switch (Svc.Prefs.get("firstSync")) {
case "resetClient":
@@ -113,9 +107,6 @@ EngineSynchronizer.prototype = {
case "wipeRemote":
this.service.wipeRemote(engineManager.enabledEngineNames);
break;
- default:
- allowEnginesHint = true;
- break;
}
if (this.service.clientsEngine.localCommands) {
@@ -145,31 +136,20 @@ EngineSynchronizer.prototype = {
try {
this._updateEnabledEngines();
} catch (ex) {
- this._log.debug("Updating enabled engines failed", ex);
+ this._log.debug("Updating enabled engines failed: " +
+ Utils.exceptionStr(ex));
this.service.errorHandler.checkServerError(ex);
this.onComplete(ex);
return;
}
- // If the engines to sync has been specified, we sync in the order specified.
- let enginesToSync;
- if (allowEnginesHint && engineNamesToSync) {
- this._log.info("Syncing specified engines", engineNamesToSync);
- enginesToSync = engineManager.get(engineNamesToSync).filter(e => e.enabled);
- } else {
- this._log.info("Syncing all enabled engines.");
- enginesToSync = engineManager.getEnabled();
- }
try {
- // We don't bother validating engines that failed to sync.
- let enginesToValidate = [];
- for (let engine of enginesToSync) {
+ for (let engine of engineManager.getEnabled()) {
// If there's any problems with syncing the engine, report the failure
if (!(this._syncEngine(engine)) || this.service.status.enforceBackoff) {
this._log.info("Aborting sync for failure in " + engine.name);
break;
}
- enginesToValidate.push(engine);
}
// If _syncEngine fails for a 401, we might not have a cluster URL here.
@@ -195,8 +175,6 @@ EngineSynchronizer.prototype = {
}
}
- Async.promiseSpinningly(this._tryValidateEngines(enginesToValidate));
-
// If there were no sync engine failures
if (this.service.status.service != SYNC_FAILED_PARTIAL) {
Svc.Prefs.set("lastSync", new Date().toString());
@@ -206,7 +184,7 @@ EngineSynchronizer.prototype = {
Svc.Prefs.reset("firstSync");
let syncTime = ((Date.now() - startTime) / 1000).toFixed(2);
- let dateStr = Utils.formatTimestamp(new Date());
+ let dateStr = new Date().toLocaleFormat(LOG_DATE_FORMAT);
this._log.info("Sync completed at " + dateStr
+ " after " + syncTime + " secs.");
}
@@ -214,106 +192,6 @@ EngineSynchronizer.prototype = {
this.onComplete(null);
},
- _tryValidateEngines: Task.async(function* (recentlySyncedEngines) {
- if (!Services.telemetry.canRecordBase || !Svc.Prefs.get("validation.enabled", false)) {
- this._log.info("Skipping validation: validation or telemetry reporting is disabled");
- return;
- }
-
- let lastValidation = Svc.Prefs.get("validation.lastTime", 0);
- let validationInterval = Svc.Prefs.get("validation.interval");
- let nowSeconds = Math.floor(Date.now() / 1000);
-
- if (nowSeconds - lastValidation < validationInterval) {
- this._log.info("Skipping validation: too recent since last validation attempt");
- return;
- }
- // Update the time now, even if we may return false still. We don't want to
- // check the rest of these more frequently than once a day.
- Svc.Prefs.set("validation.lastTime", nowSeconds);
-
- // Validation only occurs a certain percentage of the time.
- let validationProbability = Svc.Prefs.get("validation.percentageChance", 0) / 100.0;
- if (validationProbability < Math.random()) {
- this._log.info("Skipping validation: Probability threshold not met");
- return;
- }
- let maxRecords = Svc.Prefs.get("validation.maxRecords");
- if (!maxRecords) {
- // Don't bother asking the server for the counts if we know validation
- // won't happen anyway.
- return;
- }
-
- // maxRecords of -1 means "any number", so we can skip asking the server.
- // Used for tests.
- let info;
- if (maxRecords < 0) {
- info = {};
- for (let e of recentlySyncedEngines) {
- info[e.name] = 1; // needs to be < maxRecords
- }
- maxRecords = 2;
- } else {
-
- let collectionCountsURL = this.service.userBaseURL + "info/collection_counts";
- try {
- let infoResp = this.service._fetchInfo(collectionCountsURL);
- if (!infoResp.success) {
- this._log.error("Can't run validation: request to info/collection_counts responded with "
- + resp.status);
- return;
- }
- info = infoResp.obj; // might throw because obj is a getter which parses json.
- } catch (e) {
- // Not running validation is totally fine, so we just write an error log and return.
- this._log.error("Can't run validation: Caught error when fetching counts", e);
- return;
- }
- }
-
- if (!info) {
- return;
- }
-
- let engineLookup = new Map(recentlySyncedEngines.map(e => [e.name, e]));
- let toRun = [];
- for (let [engineName, recordCount] of Object.entries(info)) {
- let engine = engineLookup.get(engineName);
- if (recordCount > maxRecords || !engine) {
- this._log.debug(`Skipping validation for ${engineName} because it's not an engine or ` +
- `the number of records (${recordCount}) is greater than the maximum allowed (${maxRecords}).`);
- continue;
- }
- let validator = engine.getValidator();
- if (!validator) {
- continue;
- }
- // Put this in an array so that we know how many we're going to do, so we
- // don't tell users we're going to run some validators when we aren't.
- toRun.push({ engine, validator });
- }
-
- if (!toRun.length) {
- return;
- }
- Services.console.logStringMessage(
- "Sync is about to run a consistency check. This may be slow, and " +
- "can be controlled using the pref \"services.sync.validation.enabled\".\n" +
- "If you encounter any problems because of this, please file a bug.");
- for (let { validator, engine } of toRun) {
- try {
- let result = yield validator.validate(engine);
- Observers.notify("weave:engine:validate:finish", result, engine.name);
- } catch (e) {
- this._log.error(`Failed to run validation on ${engine.name}!`, e);
- Observers.notify("weave:engine:validate:error", e, engine.name)
- // Keep validating -- there's no reason to think that a failure for one
- // validator would mean the others will fail.
- }
- }
- }),
-
// Returns true if sync should proceed.
// false / no return value means sync should be aborted.
_syncEngine: function _syncEngine(engine) {
@@ -346,15 +224,8 @@ EngineSynchronizer.prototype = {
// If we're the only client, and no engines are marked as enabled,
// thumb our noses at the server data: it can't be right.
// Belt-and-suspenders approach to Bug 615926.
- let hasEnabledEngines = false;
- for (let e in meta.payload.engines) {
- if (e != "clients") {
- hasEnabledEngines = true;
- break;
- }
- }
-
- if ((numClients <= 1) && !hasEnabledEngines) {
+ if ((numClients <= 1) &&
+ ([e for (e in meta.payload.engines) if (e != "clients")].length == 0)) {
this._log.info("One client and no enabled engines: not touching local engine status.");
return;
}
@@ -418,7 +289,7 @@ EngineSynchronizer.prototype = {
}
// Any remaining engines were either enabled locally or disabled remotely.
- for (let engineName of enabled) {
+ for each (let engineName in enabled) {
let engine = engineManager.get(engineName);
if (Svc.Prefs.get("engineStatusChanged." + engine.prefName, false)) {
this._log.trace("The " + engineName + " engine was enabled locally.");
diff --git a/services/sync/modules/status.js b/services/sync/modules/status.js
index 100bc7965..19dff9712 100644
--- a/services/sync/modules/status.js
+++ b/services/sync/modules/status.js
@@ -4,10 +4,10 @@
this.EXPORTED_SYMBOLS = ["Status"];
-var Cc = Components.classes;
-var Ci = Components.interfaces;
-var Cr = Components.results;
-var Cu = Components.utils;
+const Cc = Components.classes;
+const Ci = Components.interfaces;
+const Cr = Components.results;
+const Cu = Components.utils;
Cu.import("resource://services-sync/constants.js");
Cu.import("resource://gre/modules/Log.jsm");
@@ -30,7 +30,10 @@ this.Status = {
.wrappedJSObject;
let idClass = service.fxAccountsEnabled ? BrowserIDManager : IdentityManager;
this.__authManager = new idClass();
- this.__authManager.initialize();
+ // .initialize returns a promise, so we need to spin until it resolves.
+ let cb = Async.makeSpinningCallback();
+ this.__authManager.initialize().then(cb, cb);
+ cb.wait();
return this.__authManager;
},
diff --git a/services/sync/modules/telemetry.js b/services/sync/modules/telemetry.js
deleted file mode 100644
index c311387f7..000000000
--- a/services/sync/modules/telemetry.js
+++ /dev/null
@@ -1,578 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this file,
- * You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-"use strict";
-
-const {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
-
-this.EXPORTED_SYMBOLS = ["SyncTelemetry"];
-
-Cu.import("resource://services-sync/browserid_identity.js");
-Cu.import("resource://services-sync/main.js");
-Cu.import("resource://services-sync/status.js");
-Cu.import("resource://services-sync/util.js");
-Cu.import("resource://services-common/observers.js");
-Cu.import("resource://services-common/async.js");
-Cu.import("resource://gre/modules/Log.jsm");
-Cu.import("resource://gre/modules/TelemetryController.jsm");
-Cu.import("resource://gre/modules/FxAccounts.jsm");
-Cu.import("resource://gre/modules/XPCOMUtils.jsm");
-Cu.import("resource://gre/modules/osfile.jsm", this);
-
-let constants = {};
-Cu.import("resource://services-sync/constants.js", constants);
-
-var fxAccountsCommon = {};
-Cu.import("resource://gre/modules/FxAccountsCommon.js", fxAccountsCommon);
-
-XPCOMUtils.defineLazyServiceGetter(this, "Telemetry",
- "@mozilla.org/base/telemetry;1",
- "nsITelemetry");
-
-const log = Log.repository.getLogger("Sync.Telemetry");
-
-const TOPICS = [
- "profile-before-change",
- "weave:service:sync:start",
- "weave:service:sync:finish",
- "weave:service:sync:error",
-
- "weave:engine:sync:start",
- "weave:engine:sync:finish",
- "weave:engine:sync:error",
- "weave:engine:sync:applied",
- "weave:engine:sync:uploaded",
- "weave:engine:validate:finish",
- "weave:engine:validate:error",
-];
-
-const PING_FORMAT_VERSION = 1;
-
-// The set of engines we record telemetry for - any other engines are ignored.
-const ENGINES = new Set(["addons", "bookmarks", "clients", "forms", "history",
- "passwords", "prefs", "tabs", "extension-storage"]);
-
-// A regex we can use to replace the profile dir in error messages. We use a
-// regexp so we can simply replace all case-insensitive occurences.
-// This escaping function is from:
-// https://developer.mozilla.org/en/docs/Web/JavaScript/Guide/Regular_Expressions
-const reProfileDir = new RegExp(
- OS.Constants.Path.profileDir.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"),
- "gi");
-
-function transformError(error, engineName) {
- if (Async.isShutdownException(error)) {
- return { name: "shutdownerror" };
- }
-
- if (typeof error === "string") {
- if (error.startsWith("error.")) {
- // This is hacky, but I can't imagine that it's not also accurate.
- return { name: "othererror", error };
- }
- // There's a chance the profiledir is in the error string which is PII we
- // want to avoid including in the ping.
- error = error.replace(reProfileDir, "[profileDir]");
- return { name: "unexpectederror", error };
- }
-
- if (error.failureCode) {
- return { name: "othererror", error: error.failureCode };
- }
-
- if (error instanceof AuthenticationError) {
- return { name: "autherror", from: error.source };
- }
-
- if (error instanceof Ci.mozIStorageError) {
- return { name: "sqlerror", code: error.result };
- }
-
- let httpCode = error.status ||
- (error.response && error.response.status) ||
- error.code;
-
- if (httpCode) {
- return { name: "httperror", code: httpCode };
- }
-
- if (error.result) {
- return { name: "nserror", code: error.result };
- }
-
- return {
- name: "unexpectederror",
- // as above, remove the profile dir value.
- error: String(error).replace(reProfileDir, "[profileDir]")
- }
-}
-
-function tryGetMonotonicTimestamp() {
- try {
- return Telemetry.msSinceProcessStart();
- } catch (e) {
- log.warn("Unable to get a monotonic timestamp!");
- return -1;
- }
-}
-
-function timeDeltaFrom(monotonicStartTime) {
- let now = tryGetMonotonicTimestamp();
- if (monotonicStartTime !== -1 && now !== -1) {
- return Math.round(now - monotonicStartTime);
- }
- return -1;
-}
-
-class EngineRecord {
- constructor(name) {
- // startTime is in ms from process start, but is monotonic (unlike Date.now())
- // so we need to keep both it and when.
- this.startTime = tryGetMonotonicTimestamp();
- this.name = name;
- }
-
- toJSON() {
- let result = Object.assign({}, this);
- delete result.startTime;
- return result;
- }
-
- finished(error) {
- let took = timeDeltaFrom(this.startTime);
- if (took > 0) {
- this.took = took;
- }
- if (error) {
- this.failureReason = transformError(error, this.name);
- }
- }
-
- recordApplied(counts) {
- if (this.incoming) {
- log.error(`Incoming records applied multiple times for engine ${this.name}!`);
- return;
- }
- if (this.name === "clients" && !counts.failed) {
- // ignore successful application of client records
- // since otherwise they show up every time and are meaningless.
- return;
- }
-
- let incomingData = {};
- let properties = ["applied", "failed", "newFailed", "reconciled"];
- // Only record non-zero properties and only record incoming at all if
- // there's at least one property we care about.
- for (let property of properties) {
- if (counts[property]) {
- incomingData[property] = counts[property];
- this.incoming = incomingData;
- }
- }
- }
-
- recordValidation(validationResult) {
- if (this.validation) {
- log.error(`Multiple validations occurred for engine ${this.name}!`);
- return;
- }
- let { problems, version, duration, recordCount } = validationResult;
- let validation = {
- version: version || 0,
- checked: recordCount || 0,
- };
- if (duration > 0) {
- validation.took = Math.round(duration);
- }
- let summarized = problems.getSummary(true).filter(({count}) => count > 0);
- if (summarized.length) {
- validation.problems = summarized;
- }
- this.validation = validation;
- }
-
- recordValidationError(e) {
- if (this.validation) {
- log.error(`Multiple validations occurred for engine ${this.name}!`);
- return;
- }
-
- this.validation = {
- failureReason: transformError(e)
- };
- }
-
- recordUploaded(counts) {
- if (counts.sent || counts.failed) {
- if (!this.outgoing) {
- this.outgoing = [];
- }
- this.outgoing.push({
- sent: counts.sent || undefined,
- failed: counts.failed || undefined,
- });
- }
- }
-}
-
-class TelemetryRecord {
- constructor(allowedEngines) {
- this.allowedEngines = allowedEngines;
- // Our failure reason. This property only exists in the generated ping if an
- // error actually occurred.
- this.failureReason = undefined;
- this.uid = "";
- this.when = Date.now();
- this.startTime = tryGetMonotonicTimestamp();
- this.took = 0; // will be set later.
-
- // All engines that have finished (ie, does not include the "current" one)
- // We omit this from the ping if it's empty.
- this.engines = [];
- // The engine that has started but not yet stopped.
- this.currentEngine = null;
- }
-
- toJSON() {
- let result = {
- when: this.when,
- uid: this.uid,
- took: this.took,
- failureReason: this.failureReason,
- status: this.status,
- deviceID: this.deviceID,
- devices: this.devices,
- };
- let engines = [];
- for (let engine of this.engines) {
- engines.push(engine.toJSON());
- }
- if (engines.length > 0) {
- result.engines = engines;
- }
- return result;
- }
-
- finished(error) {
- this.took = timeDeltaFrom(this.startTime);
- if (this.currentEngine != null) {
- log.error("Finished called for the sync before the current engine finished");
- this.currentEngine.finished(null);
- this.onEngineStop(this.currentEngine.name);
- }
- if (error) {
- this.failureReason = transformError(error);
- }
-
- // We don't bother including the "devices" field if we can't come up with a
- // UID or device ID for *this* device -- If that's the case, any data we'd
- // put there would be likely to be full of garbage anyway.
- let includeDeviceInfo = false;
- try {
- this.uid = Weave.Service.identity.hashedUID();
- let deviceID = Weave.Service.identity.deviceID();
- if (deviceID) {
- // Combine the raw device id with the metrics uid to create a stable
- // unique identifier that can't be mapped back to the user's FxA
- // identity without knowing the metrics HMAC key.
- this.deviceID = Utils.sha256(deviceID + this.uid);
- includeDeviceInfo = true;
- }
- } catch (e) {
- this.uid = "0".repeat(32);
- this.deviceID = undefined;
- }
-
- if (includeDeviceInfo) {
- let remoteDevices = Weave.Service.clientsEngine.remoteClients;
- this.devices = remoteDevices.map(device => {
- return {
- os: device.os,
- version: device.version,
- id: Utils.sha256(device.id + this.uid)
- };
- });
- }
-
- // Check for engine statuses. -- We do this now, and not in engine.finished
- // to make sure any statuses that get set "late" are recorded
- for (let engine of this.engines) {
- let status = Status.engines[engine.name];
- if (status && status !== constants.ENGINE_SUCCEEDED) {
- engine.status = status;
- }
- }
-
- let statusObject = {};
-
- let serviceStatus = Status.service;
- if (serviceStatus && serviceStatus !== constants.STATUS_OK) {
- statusObject.service = serviceStatus;
- this.status = statusObject;
- }
- let syncStatus = Status.sync;
- if (syncStatus && syncStatus !== constants.SYNC_SUCCEEDED) {
- statusObject.sync = syncStatus;
- this.status = statusObject;
- }
- }
-
- onEngineStart(engineName) {
- if (this._shouldIgnoreEngine(engineName, false)) {
- return;
- }
-
- if (this.currentEngine) {
- log.error(`Being told that engine ${engineName} has started, but current engine ${
- this.currentEngine.name} hasn't stopped`);
- // Just discard the current engine rather than making up data for it.
- }
- this.currentEngine = new EngineRecord(engineName);
- }
-
- onEngineStop(engineName, error) {
- // We only care if it's the current engine if we have a current engine.
- if (this._shouldIgnoreEngine(engineName, !!this.currentEngine)) {
- return;
- }
- if (!this.currentEngine) {
- // It's possible for us to get an error before the start message of an engine
- // (somehow), in which case we still want to record that error.
- if (!error) {
- return;
- }
- log.error(`Error triggered on ${engineName} when no current engine exists: ${error}`);
- this.currentEngine = new EngineRecord(engineName);
- }
- this.currentEngine.finished(error);
- this.engines.push(this.currentEngine);
- this.currentEngine = null;
- }
-
- onEngineApplied(engineName, counts) {
- if (this._shouldIgnoreEngine(engineName)) {
- return;
- }
- this.currentEngine.recordApplied(counts);
- }
-
- onEngineValidated(engineName, validationData) {
- if (this._shouldIgnoreEngine(engineName, false)) {
- return;
- }
- let engine = this.engines.find(e => e.name === engineName);
- if (!engine && this.currentEngine && engineName === this.currentEngine.name) {
- engine = this.currentEngine;
- }
- if (engine) {
- engine.recordValidation(validationData);
- } else {
- log.warn(`Validation event triggered for engine ${engineName}, which hasn't been synced!`);
- }
- }
-
- onEngineValidateError(engineName, error) {
- if (this._shouldIgnoreEngine(engineName, false)) {
- return;
- }
- let engine = this.engines.find(e => e.name === engineName);
- if (!engine && this.currentEngine && engineName === this.currentEngine.name) {
- engine = this.currentEngine;
- }
- if (engine) {
- engine.recordValidationError(error);
- } else {
- log.warn(`Validation failure event triggered for engine ${engineName}, which hasn't been synced!`);
- }
- }
-
- onEngineUploaded(engineName, counts) {
- if (this._shouldIgnoreEngine(engineName)) {
- return;
- }
- this.currentEngine.recordUploaded(counts);
- }
-
- _shouldIgnoreEngine(engineName, shouldBeCurrent = true) {
- if (!this.allowedEngines.has(engineName)) {
- log.info(`Notification for engine ${engineName}, but we aren't recording telemetry for it`);
- return true;
- }
- if (shouldBeCurrent) {
- if (!this.currentEngine || engineName != this.currentEngine.name) {
- log.error(`Notification for engine ${engineName} but it isn't current`);
- return true;
- }
- }
- return false;
- }
-}
-
-class SyncTelemetryImpl {
- constructor(allowedEngines) {
- log.level = Log.Level[Svc.Prefs.get("log.logger.telemetry", "Trace")];
- // This is accessible so we can enable custom engines during tests.
- this.allowedEngines = allowedEngines;
- this.current = null;
- this.setupObservers();
-
- this.payloads = [];
- this.discarded = 0;
- this.maxPayloadCount = Svc.Prefs.get("telemetry.maxPayloadCount");
- this.submissionInterval = Svc.Prefs.get("telemetry.submissionInterval") * 1000;
- this.lastSubmissionTime = Telemetry.msSinceProcessStart();
- }
-
- getPingJSON(reason) {
- return {
- why: reason,
- discarded: this.discarded || undefined,
- version: PING_FORMAT_VERSION,
- syncs: this.payloads.slice(),
- };
- }
-
- finish(reason) {
- // Note that we might be in the middle of a sync right now, and so we don't
- // want to touch this.current.
- let result = this.getPingJSON(reason);
- this.payloads = [];
- this.discarded = 0;
- this.submit(result);
- }
-
- setupObservers() {
- for (let topic of TOPICS) {
- Observers.add(topic, this, this);
- }
- }
-
- shutdown() {
- this.finish("shutdown");
- for (let topic of TOPICS) {
- Observers.remove(topic, this, this);
- }
- }
-
- submit(record) {
- // We still call submit() with possibly illegal payloads so that tests can
- // know that the ping was built. We don't end up submitting them, however.
- if (record.syncs.length) {
- log.trace(`submitting ${record.syncs.length} sync record(s) to telemetry`);
- TelemetryController.submitExternalPing("sync", record);
- }
- }
-
-
- onSyncStarted() {
- if (this.current) {
- log.warn("Observed weave:service:sync:start, but we're already recording a sync!");
- // Just discard the old record, consistent with our handling of engines, above.
- this.current = null;
- }
- this.current = new TelemetryRecord(this.allowedEngines);
- }
-
- _checkCurrent(topic) {
- if (!this.current) {
- log.warn(`Observed notification ${topic} but no current sync is being recorded.`);
- return false;
- }
- return true;
- }
-
- onSyncFinished(error) {
- if (!this.current) {
- log.warn("onSyncFinished but we aren't recording");
- return;
- }
- this.current.finished(error);
- if (this.payloads.length < this.maxPayloadCount) {
- this.payloads.push(this.current.toJSON());
- } else {
- ++this.discarded;
- }
- this.current = null;
- if ((Telemetry.msSinceProcessStart() - this.lastSubmissionTime) > this.submissionInterval) {
- this.finish("schedule");
- this.lastSubmissionTime = Telemetry.msSinceProcessStart();
- }
- }
-
- observe(subject, topic, data) {
- log.trace(`observed ${topic} ${data}`);
-
- switch (topic) {
- case "profile-before-change":
- this.shutdown();
- break;
-
- /* sync itself state changes */
- case "weave:service:sync:start":
- this.onSyncStarted();
- break;
-
- case "weave:service:sync:finish":
- if (this._checkCurrent(topic)) {
- this.onSyncFinished(null);
- }
- break;
-
- case "weave:service:sync:error":
- // argument needs to be truthy (this should always be the case)
- this.onSyncFinished(subject || "Unknown");
- break;
-
- /* engine sync state changes */
- case "weave:engine:sync:start":
- if (this._checkCurrent(topic)) {
- this.current.onEngineStart(data);
- }
- break;
- case "weave:engine:sync:finish":
- if (this._checkCurrent(topic)) {
- this.current.onEngineStop(data, null);
- }
- break;
-
- case "weave:engine:sync:error":
- if (this._checkCurrent(topic)) {
- // argument needs to be truthy (this should always be the case)
- this.current.onEngineStop(data, subject || "Unknown");
- }
- break;
-
- /* engine counts */
- case "weave:engine:sync:applied":
- if (this._checkCurrent(topic)) {
- this.current.onEngineApplied(data, subject);
- }
- break;
-
- case "weave:engine:sync:uploaded":
- if (this._checkCurrent(topic)) {
- this.current.onEngineUploaded(data, subject);
- }
- break;
-
- case "weave:engine:validate:finish":
- if (this._checkCurrent(topic)) {
- this.current.onEngineValidated(data, subject);
- }
- break;
-
- case "weave:engine:validate:error":
- if (this._checkCurrent(topic)) {
- this.current.onEngineValidateError(data, subject || "Unknown");
- }
- break;
-
- default:
- log.warn(`unexpected observer topic ${topic}`);
- break;
- }
- }
-}
-
-this.SyncTelemetry = new SyncTelemetryImpl(ENGINES);
diff --git a/services/sync/modules/userapi.js b/services/sync/modules/userapi.js
index e906440bd..ec77d63e2 100644
--- a/services/sync/modules/userapi.js
+++ b/services/sync/modules/userapi.js
@@ -8,7 +8,7 @@ this.EXPORTED_SYMBOLS = [
"UserAPI10Client",
];
-var {utils: Cu} = Components;
+const {utils: Cu} = Components;
Cu.import("resource://gre/modules/Log.jsm");
Cu.import("resource://services-common/rest.js");
diff --git a/services/sync/modules/util.js b/services/sync/modules/util.js
index e9dbcb37d..67cc3f063 100644
--- a/services/sync/modules/util.js
+++ b/services/sync/modules/util.js
@@ -4,7 +4,7 @@
this.EXPORTED_SYMBOLS = ["XPCOMUtils", "Services", "Utils", "Async", "Svc", "Str"];
-var {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components;
+const {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components;
Cu.import("resource://gre/modules/Log.jsm");
Cu.import("resource://services-common/observers.js");
@@ -35,6 +35,8 @@ this.Utils = {
// In the ideal world, references to these would be removed.
nextTick: CommonUtils.nextTick,
namedTimer: CommonUtils.namedTimer,
+ exceptionStr: CommonUtils.exceptionStr,
+ stackTrace: CommonUtils.stackTrace,
makeURI: CommonUtils.makeURI,
encodeUTF8: CommonUtils.encodeUTF8,
decodeUTF8: CommonUtils.decodeUTF8,
@@ -52,7 +54,6 @@ this.Utils = {
digestBytes: CryptoUtils.digestBytes,
sha1: CryptoUtils.sha1,
sha1Base32: CryptoUtils.sha1Base32,
- sha256: CryptoUtils.sha256,
makeHMACKey: CryptoUtils.makeHMACKey,
makeHMACHasher: CryptoUtils.makeHMACHasher,
hkdfExpand: CryptoUtils.hkdfExpand,
@@ -61,25 +62,6 @@ this.Utils = {
getHTTPMACSHA1Header: CryptoUtils.getHTTPMACSHA1Header,
/**
- * The string to use as the base User-Agent in Sync requests.
- * This string will look something like
- *
- * Firefox/49.0a1 (Windows NT 6.1; WOW64; rv:46.0) FxSync/1.51.0.20160516142357.desktop
- */
- _userAgent: null,
- get userAgent() {
- if (!this._userAgent) {
- let hph = Cc["@mozilla.org/network/protocol;1?name=http"].getService(Ci.nsIHttpProtocolHandler);
- this._userAgent =
- Services.appinfo.name + "/" + Services.appinfo.version + // Product.
- " (" + hph.oscpu + ")" + // (oscpu)
- " FxSync/" + WEAVE_VERSION + "." + // Sync.
- Services.appinfo.appBuildID + "."; // Build.
- }
- return this._userAgent + Svc.Prefs.get("client.type", "desktop");
- },
-
- /**
* Wrap a function to catch all exceptions and log them
*
* @usage MyObj._catch = Utils.catch;
@@ -95,7 +77,7 @@ this.Utils = {
return func.call(thisArg);
}
catch(ex) {
- thisArg._log.debug("Exception calling " + (func.name || "anonymous function"), ex);
+ thisArg._log.debug("Exception: " + Utils.exceptionStr(ex));
if (exceptionCallback) {
return exceptionCallback.call(thisArg, ex);
}
@@ -271,14 +253,14 @@ this.Utils = {
*/
base32ToFriendly: function base32ToFriendly(input) {
return input.toLowerCase()
- .replace(/l/g, '8')
- .replace(/o/g, '9');
+ .replace("l", '8', "g")
+ .replace("o", '9', "g");
},
base32FromFriendly: function base32FromFriendly(input) {
return input.toUpperCase()
- .replace(/8/g, 'L')
- .replace(/9/g, 'O');
+ .replace("8", 'L', "g")
+ .replace("9", 'O', "g");
},
/**
@@ -356,14 +338,12 @@ this.Utils = {
try {
json = yield CommonUtils.readJSON(path);
+ } catch (e if e instanceof OS.File.Error && e.becauseNoSuchFile) {
+ // Ignore non-existent files.
} catch (e) {
- if (e instanceof OS.File.Error && e.becauseNoSuchFile) {
- // Ignore non-existent files, but explicitly return null.
- json = null;
- } else {
- if (that._log) {
- that._log.debug("Failed to load json", e);
- }
+ if (that._log) {
+ that._log.debug("Failed to load json: " +
+ CommonUtils.exceptionStr(e));
}
}
@@ -411,52 +391,6 @@ this.Utils = {
}
}),
- /**
- * Move a json file in the profile directory. Will fail if a file exists at the
- * destination.
- *
- * @returns a promise that resolves to undefined on success, or rejects on failure
- *
- * @param aFrom
- * Current path to the JSON file saved on disk, relative to profileDir/weave
- * .json will be appended to the file name.
- * @param aTo
- * New path to the JSON file saved on disk, relative to profileDir/weave
- * .json will be appended to the file name.
- * @param that
- * Object to use for logging
- */
- jsonMove(aFrom, aTo, that) {
- let pathFrom = OS.Path.join(OS.Constants.Path.profileDir, "weave",
- ...(aFrom + ".json").split("/"));
- let pathTo = OS.Path.join(OS.Constants.Path.profileDir, "weave",
- ...(aTo + ".json").split("/"));
- if (that._log) {
- that._log.trace("Moving " + pathFrom + " to " + pathTo);
- }
- return OS.File.move(pathFrom, pathTo, { noOverwrite: true });
- },
-
- /**
- * Removes a json file in the profile directory.
- *
- * @returns a promise that resolves to undefined on success, or rejects on failure
- *
- * @param filePath
- * Current path to the JSON file saved on disk, relative to profileDir/weave
- * .json will be appended to the file name.
- * @param that
- * Object to use for logging
- */
- jsonRemove(filePath, that) {
- let path = OS.Path.join(OS.Constants.Path.profileDir, "weave",
- ...(filePath + ".json").split("/"));
- if (that._log) {
- that._log.trace("Deleting " + path);
- }
- return OS.File.remove(path, { ignoreAbsent: true });
- },
-
getErrorString: function Utils_getErrorString(error, args) {
try {
return Str.errors.get(error, args || null);
@@ -543,7 +477,7 @@ this.Utils = {
// 20-char sync key.
if (pp.length == 23 &&
- [5, 11, 17].every(i => pp[i] == '-')) {
+ [5, 11, 17].every(function(i) pp[i] == '-')) {
return pp.slice(0, 5) + pp.slice(6, 11)
+ pp.slice(12, 17) + pp.slice(18, 23);
@@ -551,7 +485,7 @@ this.Utils = {
// "Modern" 26-char key.
if (pp.length == 31 &&
- [1, 7, 13, 19, 25].every(i => pp[i] == '-')) {
+ [1, 7, 13, 19, 25].every(function(i) pp[i] == '-')) {
return pp.slice(0, 1) + pp.slice(2, 7)
+ pp.slice(8, 13) + pp.slice(14, 19)
@@ -681,12 +615,30 @@ this.Utils = {
* Get the FxA identity hosts.
*/
getSyncCredentialsHostsFxA: function() {
+ // This is somewhat expensive and the result static, so we cache the result.
+ if (this._syncCredentialsHostsFxA) {
+ return this._syncCredentialsHostsFxA;
+ }
let result = new Set();
// the FxA host
result.add(FxAccountsCommon.FXA_PWDMGR_HOST);
- // We used to include the FxA hosts (hence the Set() result) but we now
- // don't give them special treatment (hence the Set() with exactly 1 item)
- return result;
+ //
+ // The FxA hosts - these almost certainly all have the same hostname, but
+ // better safe than sorry...
+ for (let prefName of ["identity.fxaccounts.remote.force_auth.uri",
+ "identity.fxaccounts.remote.signup.uri",
+ "identity.fxaccounts.remote.signin.uri",
+ "identity.fxaccounts.settings.uri"]) {
+ let prefVal;
+ try {
+ prefVal = Services.prefs.getCharPref(prefName);
+ } catch (_) {
+ continue;
+ }
+ let uri = Services.io.newURI(prefVal, null, null);
+ result.add(uri.prePath);
+ }
+ return this._syncCredentialsHostsFxA = result;
},
getDefaultDeviceName() {
@@ -720,32 +672,6 @@ this.Utils = {
Cc["@mozilla.org/network/protocol;1?name=http"].getService(Ci.nsIHttpProtocolHandler).oscpu;
return Str.sync.get("client.name2", [user, appName, system]);
- },
-
- getDeviceName() {
- const deviceName = Svc.Prefs.get("client.name", "");
-
- if (deviceName === "") {
- return this.getDefaultDeviceName();
- }
-
- return deviceName;
- },
-
- getDeviceType() {
- return Svc.Prefs.get("client.type", DEVICE_TYPE_DESKTOP);
- },
-
- formatTimestamp(date) {
- // Format timestamp as: "%Y-%m-%d %H:%M:%S"
- let year = String(date.getFullYear());
- let month = String(date.getMonth() + 1).padStart(2, "0");
- let day = String(date.getDate()).padStart(2, "0");
- let hours = String(date.getHours()).padStart(2, "0");
- let minutes = String(date.getMinutes()).padStart(2, "0");
- let seconds = String(date.getSeconds()).padStart(2, "0");
-
- return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}`;
}
};
@@ -764,7 +690,7 @@ Svc.Prefs = new Preferences(PREFS_BRANCH);
Svc.DefaultPrefs = new Preferences({branch: PREFS_BRANCH, defaultBranch: true});
Svc.Obs = Observers;
-var _sessionCID = Services.appinfo.ID == SEAMONKEY_ID ?
+let _sessionCID = Services.appinfo.ID == SEAMONKEY_ID ?
"@mozilla.org/suite/sessionstore;1" :
"@mozilla.org/browser/sessionstore;1";