diff options
Diffstat (limited to 'services/sync/modules')
35 files changed, 2602 insertions, 5123 deletions
diff --git a/services/sync/modules/FxaMigrator.jsm b/services/sync/modules/FxaMigrator.jsm new file mode 100644 index 000000000..605ee5d7f --- /dev/null +++ b/services/sync/modules/FxaMigrator.jsm @@ -0,0 +1,546 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict;" + +const {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components; + +Cu.import("resource://gre/modules/XPCOMUtils.jsm"); +Cu.import("resource://gre/modules/Log.jsm"); +Cu.import("resource://gre/modules/Services.jsm"); +Cu.import("resource://gre/modules/Task.jsm"); + +XPCOMUtils.defineLazyModuleGetter(this, "fxAccounts", + "resource://gre/modules/FxAccounts.jsm"); + +XPCOMUtils.defineLazyGetter(this, "WeaveService", function() { + return Cc["@mozilla.org/weave/service;1"] + .getService(Components.interfaces.nsISupports) + .wrappedJSObject; +}); + +XPCOMUtils.defineLazyModuleGetter(this, "Weave", + "resource://services-sync/main.js"); + +// FxAccountsCommon.js doesn't use a "namespace", so create one here. +let fxAccountsCommon = {}; +Cu.import("resource://gre/modules/FxAccountsCommon.js", fxAccountsCommon); + +// We send this notification whenever the "user" migration state changes. +const OBSERVER_STATE_CHANGE_TOPIC = "fxa-migration:state-changed"; +// We also send the state notification when we *receive* this. This allows +// consumers to avoid loading this module until it receives a notification +// from us (which may never happen if there's no migration to do) +const OBSERVER_STATE_REQUEST_TOPIC = "fxa-migration:state-request"; + +// We send this notification whenever the migration is paused waiting for +// something internal to complete. +const OBSERVER_INTERNAL_STATE_CHANGE_TOPIC = "fxa-migration:internal-state-changed"; + +// We use this notification so Sync's healthreport module can record telemetry +// (actually via "health report") for us. +const OBSERVER_INTERNAL_TELEMETRY_TOPIC = "fxa-migration:internal-telemetry"; + +const OBSERVER_TOPICS = [ + "xpcom-shutdown", + "weave:service:sync:start", + "weave:service:sync:finish", + "weave:service:sync:error", + "weave:eol", + OBSERVER_STATE_REQUEST_TOPIC, + fxAccountsCommon.ONLOGIN_NOTIFICATION, + fxAccountsCommon.ONLOGOUT_NOTIFICATION, + fxAccountsCommon.ONVERIFIED_NOTIFICATION, +]; + +// A list of preference names we write to the migration sentinel. We only +// write ones that have a user-set value. +const FXA_SENTINEL_PREFS = [ + "identity.fxaccounts.auth.uri", + "identity.fxaccounts.remote.force_auth.uri", + "identity.fxaccounts.remote.signup.uri", + "identity.fxaccounts.remote.signin.uri", + "identity.fxaccounts.settings.uri", + "services.sync.tokenServerURI", +]; + +function Migrator() { + // Leave the log-level as Debug - Sync will setup log appenders such that + // these messages generally will not be seen unless other log related + // prefs are set. + this.log.level = Log.Level.Debug; + + this._nextUserStatePromise = Promise.resolve(); + + for (let topic of OBSERVER_TOPICS) { + Services.obs.addObserver(this, topic, false); + } + // ._state is an optimization so we avoid sending redundant observer + // notifications when the state hasn't actually changed. + this._state = null; +} + +Migrator.prototype = { + log: Log.repository.getLogger("Sync.SyncMigration"), + + // What user action is necessary to push the migration forward? + // A |null| state means there is nothing to do. Note that a null state implies + // either. (a) no migration is necessary or (b) that the migrator module is + // waiting for something outside of the user's control - eg, sync to complete, + // the migration sentinel to be uploaded, etc. In most cases the wait will be + // short, but edge cases (eg, no network, sync bugs that prevent it stopping + // until shutdown) may require a significantly longer wait. + STATE_USER_FXA: "waiting for user to be signed in to FxA", + STATE_USER_FXA_VERIFIED: "waiting for a verified FxA user", + + // What internal state are we at? This is primarily used for FHR reporting so + // we can determine why exactly we might be stalled. + STATE_INTERNAL_WAITING_SYNC_COMPLETE: "waiting for sync to complete", + STATE_INTERNAL_WAITING_WRITE_SENTINEL: "waiting for sentinel to be written", + STATE_INTERNAL_WAITING_START_OVER: "waiting for sync to reset itself", + STATE_INTERNAL_COMPLETE: "migration complete", + + // Flags for the telemetry we record. The UI will call a helper to record + // the fact some UI was interacted with. + TELEMETRY_ACCEPTED: "accepted", + TELEMETRY_DECLINED: "declined", + TELEMETRY_UNLINKED: "unlinked", + + finalize() { + for (let topic of OBSERVER_TOPICS) { + Services.obs.removeObserver(this, topic); + } + }, + + observe(subject, topic, data) { + this.log.debug("observed " + topic); + switch (topic) { + case "xpcom-shutdown": + this.finalize(); + break; + + case OBSERVER_STATE_REQUEST_TOPIC: + // someone has requested the state - send it. + this._queueCurrentUserState(true); + break; + + default: + // some other observer that may affect our state has fired, so update. + this._queueCurrentUserState().then( + () => this.log.debug("update state from observer " + topic + " complete") + ).catch(err => { + let msg = "Failed to handle topic " + topic + ": " + err; + Cu.reportError(msg); + this.log.error(msg); + }); + } + }, + + // Try and move to a state where we are blocked on a user action. + // This needs to be restartable, and the states may, in edge-cases, end + // up going backwards (eg, user logs out while we are waiting to be told + // about verification) + // This is called by our observer notifications - so if there is already + // a promise in-flight, it's possible we will miss something important - so + // we wait for the in-flight one to complete then fire another (ie, this + // is effectively a queue of promises) + _queueCurrentUserState(forceObserver = false) { + return this._nextUserStatePromise = this._nextUserStatePromise.then( + () => this._promiseCurrentUserState(forceObserver), + err => { + let msg = "Failed to determine the current user state: " + err; + Cu.reportError(msg); + this.log.error(msg); + return this._promiseCurrentUserState(forceObserver) + } + ); + }, + + _promiseCurrentUserState: Task.async(function* (forceObserver) { + this.log.trace("starting _promiseCurrentUserState"); + let update = (newState, email=null) => { + this.log.info("Migration state: '${state}' => '${newState}'", + {state: this._state, newState: newState}); + if (forceObserver || newState !== this._state) { + this._state = newState; + let subject = Cc["@mozilla.org/supports-string;1"] + .createInstance(Ci.nsISupportsString); + subject.data = email || ""; + Services.obs.notifyObservers(subject, OBSERVER_STATE_CHANGE_TOPIC, newState); + } + return newState; + } + + // If we have no sync user, or are already using an FxA account we must + // be done. + if (WeaveService.fxAccountsEnabled) { + // should not be necessary, but if we somehow ended up with FxA enabled + // and sync blocked it would be bad - so better safe than sorry. + this.log.debug("FxA enabled - there's nothing to do!") + this._unblockSync(); + return update(null); + } + + // so we need to migrate - let's see how far along we are. + // If sync isn't in EOL mode, then we are still waiting for the server + // to offer the migration process - so no user action necessary. + let isEOL = false; + try { + isEOL = !!Services.prefs.getCharPref("services.sync.errorhandler.alert.mode"); + } catch (e) {} + + if (!isEOL) { + return update(null); + } + + // So we are in EOL mode - have we a user? + let fxauser = yield fxAccounts.getSignedInUser(); + if (!fxauser) { + // See if there is a migration sentinel so we can send the email + // address that was used on a different device for this account (ie, if + // this is a "join the party" migration rather than the first) + let sentinel = yield this._getSyncMigrationSentinel(); + return update(this.STATE_USER_FXA, sentinel && sentinel.email); + } + if (!fxauser.verified) { + return update(this.STATE_USER_FXA_VERIFIED, fxauser.email); + } + + // So we just have housekeeping to do - we aren't blocked on a user, so + // reflect that. + this.log.info("No next user state - doing some housekeeping"); + update(null); + + // We need to disable sync from automatically starting, + // and if we are currently syncing wait for it to complete. + this._blockSync(); + + // Are we currently syncing? + if (Weave.Service._locked) { + // our observers will kick us further along when complete. + this.log.info("waiting for sync to complete") + Services.obs.notifyObservers(null, OBSERVER_INTERNAL_STATE_CHANGE_TOPIC, + this.STATE_INTERNAL_WAITING_SYNC_COMPLETE); + return null; + } + + // Write the migration sentinel if necessary. + Services.obs.notifyObservers(null, OBSERVER_INTERNAL_STATE_CHANGE_TOPIC, + this.STATE_INTERNAL_WAITING_WRITE_SENTINEL); + yield this._setMigrationSentinelIfNecessary(); + + // Get the list of enabled engines to we can restore that state. + let enginePrefs = this._getEngineEnabledPrefs(); + + // Must be ready to perform the actual migration. + this.log.info("Performing final sync migration steps"); + // Do the actual migration. We setup one observer for when the new identity + // is about to be initialized so we can reset some key preferences - but + // there's no promise associated with this. + let observeStartOverIdentity; + Services.obs.addObserver(observeStartOverIdentity = () => { + this.log.info("observed that startOver is about to re-initialize the identity"); + Services.obs.removeObserver(observeStartOverIdentity, "weave:service:start-over:init-identity"); + // We've now reset all sync prefs - set the engine related prefs back to + // what they were. + for (let [prefName, prefType, prefVal] of enginePrefs) { + this.log.debug("Restoring pref ${prefName} (type=${prefType}) to ${prefVal}", + {prefName, prefType, prefVal}); + switch (prefType) { + case Services.prefs.PREF_BOOL: + Services.prefs.setBoolPref(prefName, prefVal); + break; + case Services.prefs.PREF_STRING: + Services.prefs.setCharPref(prefName, prefVal); + break; + default: + // _getEngineEnabledPrefs doesn't return any other type... + Cu.reportError("unknown engine pref type for " + prefName + ": " + prefType); + } + } + }, "weave:service:start-over:init-identity", false); + + // And another observer for the startOver being fully complete - the only + // reason for this is so we can wait until everything is fully reset. + let startOverComplete = new Promise((resolve, reject) => { + let observe; + Services.obs.addObserver(observe = () => { + this.log.info("observed that startOver is complete"); + Services.obs.removeObserver(observe, "weave:service:start-over:finish"); + resolve(); + }, "weave:service:start-over:finish", false); + }); + + Weave.Service.startOver(); + // need to wait for an observer. + Services.obs.notifyObservers(null, OBSERVER_INTERNAL_STATE_CHANGE_TOPIC, + this.STATE_INTERNAL_WAITING_START_OVER); + yield startOverComplete; + // observer fired, now kick things off with the FxA user. + this.log.info("scheduling initial FxA sync."); + // Note we technically don't need to unblockSync as by now all sync prefs + // have been reset - but it doesn't hurt. + this._unblockSync(); + Weave.Service.scheduler.scheduleNextSync(0); + + // Tell the front end that migration is now complete -- Sync is now + // configured with an FxA user. + forceObserver = true; + this.log.info("Migration complete"); + update(null); + + Services.obs.notifyObservers(null, OBSERVER_INTERNAL_STATE_CHANGE_TOPIC, + this.STATE_INTERNAL_COMPLETE); + return null; + }), + + /* Return an object with the preferences we care about */ + _getSentinelPrefs() { + let result = {}; + for (let pref of FXA_SENTINEL_PREFS) { + if (Services.prefs.prefHasUserValue(pref)) { + result[pref] = Services.prefs.getCharPref(pref); + } + } + return result; + }, + + /* Apply any preferences we've obtained from the sentinel */ + _applySentinelPrefs(savedPrefs) { + for (let pref of FXA_SENTINEL_PREFS) { + if (savedPrefs[pref]) { + Services.prefs.setCharPref(pref, savedPrefs[pref]); + } + } + }, + + /* Ask sync to upload the migration sentinel */ + _setSyncMigrationSentinel: Task.async(function* () { + yield WeaveService.whenLoaded(); + let signedInUser = yield fxAccounts.getSignedInUser(); + let sentinel = { + email: signedInUser.email, + uid: signedInUser.uid, + verified: signedInUser.verified, + prefs: this._getSentinelPrefs(), + }; + yield Weave.Service.setFxAMigrationSentinel(sentinel); + }), + + /* Ask sync to upload the migration sentinal if we (or any other linked device) + haven't previously written one. + */ + _setMigrationSentinelIfNecessary: Task.async(function* () { + if (!(yield this._getSyncMigrationSentinel())) { + this.log.info("writing the migration sentinel"); + yield this._setSyncMigrationSentinel(); + } + }), + + /* Ask sync to return a migration sentinel if one exists, otherwise return null */ + _getSyncMigrationSentinel: Task.async(function* () { + yield WeaveService.whenLoaded(); + let sentinel = yield Weave.Service.getFxAMigrationSentinel(); + this.log.debug("got migration sentinel ${}", sentinel); + return sentinel; + }), + + _getDefaultAccountName: Task.async(function* (sentinel) { + // Requires looking to see if other devices have written a migration + // sentinel (eg, see _haveSynchedMigrationSentinel), and if not, see if + // the legacy account name appears to be a valid email address (via the + // services.sync.account pref), otherwise return null. + // NOTE: Sync does all this synchronously via nested event loops, but we + // expose a promise to make future migration to an async-sync easier. + if (sentinel && sentinel.email) { + this.log.info("defaultAccountName found via sentinel: ${}", sentinel.email); + return sentinel.email; + } + // No previous migrations, so check the existing account name. + let account = Weave.Service.identity.account; + if (account && account.contains("@")) { + this.log.info("defaultAccountName found via legacy account name: {}", account); + return account; + } + this.log.info("defaultAccountName could not find an account"); + return null; + }), + + // Prevent sync from automatically starting + _blockSync() { + Weave.Service.scheduler.blockSync(); + }, + + _unblockSync() { + Weave.Service.scheduler.unblockSync(); + }, + + /* Return a list of [prefName, prefType, prefVal] for all engine related + preferences. + */ + _getEngineEnabledPrefs() { + let result = []; + for (let engine of Weave.Service.engineManager.getAll()) { + let prefName = "services.sync.engine." + engine.prefName; + let prefVal; + try { + prefVal = Services.prefs.getBoolPref(prefName); + result.push([prefName, Services.prefs.PREF_BOOL, prefVal]); + } catch (ex) {} /* just skip this pref */ + } + // and the declined list. + try { + let prefName = "services.sync.declinedEngines"; + let prefVal = Services.prefs.getCharPref(prefName); + result.push([prefName, Services.prefs.PREF_STRING, prefVal]); + } catch (ex) {} + return result; + }, + + /* return true if all engines are enabled, false otherwise. */ + _allEnginesEnabled() { + return Weave.Service.engineManager.getAll().every(e => e.enabled); + }, + + /* + * Some helpers for the UI to try and move to the next state. + */ + + // Open a UI for the user to create a Firefox Account. This should only be + // called while we are in the STATE_USER_FXA state. When the user completes + // the creation we'll see an ONLOGIN_NOTIFICATION notification from FxA and + // we'll move to either the STATE_USER_FXA_VERIFIED state or we'll just + // complete the migration if they login as an already verified user. + createFxAccount: Task.async(function* (win) { + let {url, options} = yield this.getFxAccountCreationOptions(); + win.switchToTabHavingURI(url, true, options); + // An FxA observer will fire when the user completes this, which will + // cause us to move to the next "user blocked" state and notify via our + // observer notification. + }), + + // Returns an object with properties "url" and "options", suitable for + // opening FxAccounts to create/signin to FxA suitable for the migration + // state. The caller of this is responsible for the actual opening of the + // page. + // This should only be called while we are in the STATE_USER_FXA state. When + // the user completes the creation we'll see an ONLOGIN_NOTIFICATION + // notification from FxA and we'll move to either the STATE_USER_FXA_VERIFIED + // state or we'll just complete the migration if they login as an already + // verified user. + getFxAccountCreationOptions: Task.async(function* (win) { + // warn if we aren't in the expected state - but go ahead anyway! + if (this._state != this.STATE_USER_FXA) { + this.log.warn("getFxAccountCreationOptions called in an unexpected state: ${}", this._state); + } + // We need to obtain the sentinel and apply any prefs that might be + // specified *before* attempting to setup FxA as the prefs might + // specify custom servers etc. + let sentinel = yield this._getSyncMigrationSentinel(); + if (sentinel && sentinel.prefs) { + this._applySentinelPrefs(sentinel.prefs); + } + // If we already have a sentinel then we assume the user has previously + // created the specified account, so just ask to sign-in. + let action = sentinel ? "signin" : "signup"; + // See if we can find a default account name to use. + let email = yield this._getDefaultAccountName(sentinel); + let tail = email ? "&email=" + encodeURIComponent(email) : ""; + // A special flag so server-side metrics can tell this is part of migration. + tail += "&migration=sync11"; + // We want to ask FxA to offer a "Customize Sync" checkbox iff any engines + // are disabled. + let customize = !this._allEnginesEnabled(); + tail += "&customizeSync=" + customize; + + // We assume the caller of this is going to actually use it, so record + // telemetry now. + this.recordTelemetry(this.TELEMETRY_ACCEPTED); + return { + url: "about:accounts?action=" + action + tail, + options: {ignoreFragment: true, replaceQueryString: true} + }; + }), + + // Ask the FxA servers to re-send a verification mail for the currently + // logged in user. This should only be called while we are in the + // STATE_USER_FXA_VERIFIED state. When the user clicks on the link in + // the mail we should see an ONVERIFIED_NOTIFICATION which will cause us + // to complete the migration. + resendVerificationMail: Task.async(function * (win) { + // warn if we aren't in the expected state - but go ahead anyway! + if (this._state != this.STATE_USER_FXA_VERIFIED) { + this.log.warn("resendVerificationMail called in an unexpected state: ${}", this._state); + } + let ok = true; + try { + yield fxAccounts.resendVerificationEmail(); + } catch (ex) { + this.log.error("Failed to resend verification mail: ${}", ex); + ok = false; + } + this.recordTelemetry(this.TELEMETRY_ACCEPTED); + let fxauser = yield fxAccounts.getSignedInUser(); + let sb = Services.strings.createBundle("chrome://browser/locale/accounts.properties"); + + let heading = ok ? + sb.formatStringFromName("verificationSentHeading", [fxauser.email], 1) : + sb.GetStringFromName("verificationNotSentHeading"); + let title = sb.GetStringFromName(ok ? "verificationSentTitle" : "verificationNotSentTitle"); + let description = sb.GetStringFromName(ok ? "verificationSentDescription" + : "verificationNotSentDescription"); + + let factory = Cc["@mozilla.org/prompter;1"] + .getService(Ci.nsIPromptFactory); + let prompt = factory.getPrompt(win, Ci.nsIPrompt); + let bag = prompt.QueryInterface(Ci.nsIWritablePropertyBag2); + bag.setPropertyAsBool("allowTabModal", true); + + prompt.alert(title, heading + "\n\n" + description); + }), + + // "forget" about the current Firefox account. This should only be called + // while we are in the STATE_USER_FXA_VERIFIED state. After this we will + // see an ONLOGOUT_NOTIFICATION, which will cause the migrator to return back + // to the STATE_USER_FXA state, from where they can choose a different account. + forgetFxAccount: Task.async(function * () { + // warn if we aren't in the expected state - but go ahead anyway! + if (this._state != this.STATE_USER_FXA_VERIFIED) { + this.log.warn("forgetFxAccount called in an unexpected state: ${}", this._state); + } + return fxAccounts.signOut(); + }), + + recordTelemetry(flag) { + // Note the value is the telemetry field name - but this is an + // implementation detail which could be changed later. + switch (flag) { + case this.TELEMETRY_ACCEPTED: + case this.TELEMETRY_UNLINKED: + case this.TELEMETRY_DECLINED: + Services.obs.notifyObservers(null, OBSERVER_INTERNAL_TELEMETRY_TOPIC, flag); + break; + default: + throw new Error("Unexpected telemetry flag: " + flag); + } + }, + + get learnMoreLink() { + try { + var url = Services.prefs.getCharPref("app.support.baseURL"); + } catch (err) { + return null; + } + url += "sync-upgrade"; + let sb = Services.strings.createBundle("chrome://weave/locale/services/sync.properties"); + return { + text: sb.GetStringFromName("sync.eol.learnMore.label"), + href: Services.urlFormatter.formatURL(url), + }; + }, +}; + +// We expose a singleton +this.EXPORTED_SYMBOLS = ["fxaMigrator"]; +let fxaMigrator = new Migrator(); diff --git a/services/sync/modules/SyncedTabs.jsm b/services/sync/modules/SyncedTabs.jsm deleted file mode 100644 index 1a69e3564..000000000 --- a/services/sync/modules/SyncedTabs.jsm +++ /dev/null @@ -1,301 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -"use strict"; - -this.EXPORTED_SYMBOLS = ["SyncedTabs"]; - - -const { classes: Cc, interfaces: Ci, results: Cr, utils: Cu } = Components; - -Cu.import("resource://gre/modules/Services.jsm"); -Cu.import("resource://gre/modules/XPCOMUtils.jsm"); -Cu.import("resource://gre/modules/Task.jsm"); -Cu.import("resource://gre/modules/Log.jsm"); -Cu.import("resource://gre/modules/PlacesUtils.jsm", this); -Cu.import("resource://services-sync/main.js"); -Cu.import("resource://gre/modules/Preferences.jsm"); - -// The Sync XPCOM service -XPCOMUtils.defineLazyGetter(this, "weaveXPCService", function() { - return Cc["@mozilla.org/weave/service;1"] - .getService(Ci.nsISupports) - .wrappedJSObject; -}); - -// from MDN... -function escapeRegExp(string) { - return string.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); -} - -// A topic we fire whenever we have new tabs available. This might be due -// to a request made by this module to refresh the tab list, or as the result -// of a regularly scheduled sync. The intent is that consumers just listen -// for this notification and update their UI in response. -const TOPIC_TABS_CHANGED = "services.sync.tabs.changed"; - -// The interval, in seconds, before which we consider the existing list -// of tabs "fresh enough" and don't force a new sync. -const TABS_FRESH_ENOUGH_INTERVAL = 30; - -let log = Log.repository.getLogger("Sync.RemoteTabs"); -// A new scope to do the logging thang... -(function() { - let level = Preferences.get("services.sync.log.logger.tabs"); - if (level) { - let appender = new Log.DumpAppender(); - log.level = appender.level = Log.Level[level] || Log.Level.Debug; - log.addAppender(appender); - } -})(); - - -// A private singleton that does the work. -let SyncedTabsInternal = { - /* Make a "tab" record. Returns a promise */ - _makeTab: Task.async(function* (client, tab, url, showRemoteIcons) { - let icon; - if (showRemoteIcons) { - icon = tab.icon; - } - if (!icon) { - try { - icon = (yield PlacesUtils.promiseFaviconLinkUrl(url)).spec; - } catch (ex) { /* no favicon avaiable */ } - } - if (!icon) { - icon = ""; - } - return { - type: "tab", - title: tab.title || url, - url, - icon, - client: client.id, - lastUsed: tab.lastUsed, - }; - }), - - /* Make a "client" record. Returns a promise for consistency with _makeTab */ - _makeClient: Task.async(function* (client) { - return { - id: client.id, - type: "client", - name: Weave.Service.clientsEngine.getClientName(client.id), - isMobile: Weave.Service.clientsEngine.isMobile(client.id), - lastModified: client.lastModified * 1000, // sec to ms - tabs: [] - }; - }), - - _tabMatchesFilter(tab, filter) { - let reFilter = new RegExp(escapeRegExp(filter), "i"); - return tab.url.match(reFilter) || tab.title.match(reFilter); - }, - - getTabClients: Task.async(function* (filter) { - log.info("Generating tab list with filter", filter); - let result = []; - - // If Sync isn't ready, don't try and get anything. - if (!weaveXPCService.ready) { - log.debug("Sync isn't yet ready, so returning an empty tab list"); - return result; - } - - // A boolean that controls whether we should show the icon from the remote tab. - const showRemoteIcons = Preferences.get("services.sync.syncedTabs.showRemoteIcons", true); - - let engine = Weave.Service.engineManager.get("tabs"); - - let seenURLs = new Set(); - let parentIndex = 0; - let ntabs = 0; - - for (let [guid, client] of Object.entries(engine.getAllClients())) { - if (!Weave.Service.clientsEngine.remoteClientExists(client.id)) { - continue; - } - let clientRepr = yield this._makeClient(client); - log.debug("Processing client", clientRepr); - - for (let tab of client.tabs) { - let url = tab.urlHistory[0]; - log.debug("remote tab", url); - // Note there are some issues with tracking "seen" tabs, including: - // * We really can't return the entire urlHistory record as we are - // only checking the first entry - others might be different. - // * We don't update the |lastUsed| timestamp to reflect the - // most-recently-seen time. - // In a followup we should consider simply dropping this |seenUrls| - // check and return duplicate records - it seems the user will be more - // confused by tabs not showing up on a device (because it was detected - // as a dupe so it only appears on a different device) than being - // confused by seeing the same tab on different clients. - if (!url || seenURLs.has(url)) { - continue; - } - let tabRepr = yield this._makeTab(client, tab, url, showRemoteIcons); - if (filter && !this._tabMatchesFilter(tabRepr, filter)) { - continue; - } - seenURLs.add(url); - clientRepr.tabs.push(tabRepr); - } - // We return all clients, even those without tabs - the consumer should - // filter it if they care. - ntabs += clientRepr.tabs.length; - result.push(clientRepr); - } - log.info(`Final tab list has ${result.length} clients with ${ntabs} tabs.`); - return result; - }), - - syncTabs(force) { - if (!force) { - // Don't bother refetching tabs if we already did so recently - let lastFetch = Preferences.get("services.sync.lastTabFetch", 0); - let now = Math.floor(Date.now() / 1000); - if (now - lastFetch < TABS_FRESH_ENOUGH_INTERVAL) { - log.info("_refetchTabs was done recently, do not doing it again"); - return Promise.resolve(false); - } - } - - // If Sync isn't configured don't try and sync, else we will get reports - // of a login failure. - if (Weave.Status.checkSetup() == Weave.CLIENT_NOT_CONFIGURED) { - log.info("Sync client is not configured, so not attempting a tab sync"); - return Promise.resolve(false); - } - // Ask Sync to just do the tabs engine if it can. - // Sync is currently synchronous, so do it after an event-loop spin to help - // keep the UI responsive. - return new Promise((resolve, reject) => { - Services.tm.currentThread.dispatch(() => { - try { - log.info("Doing a tab sync."); - Weave.Service.sync(["tabs"]); - resolve(true); - } catch (ex) { - log.error("Sync failed", ex); - reject(ex); - }; - }, Ci.nsIThread.DISPATCH_NORMAL); - }); - }, - - observe(subject, topic, data) { - log.trace(`observed topic=${topic}, data=${data}, subject=${subject}`); - switch (topic) { - case "weave:engine:sync:finish": - if (data != "tabs") { - return; - } - // The tabs engine just finished syncing - // Set our lastTabFetch pref here so it tracks both explicit sync calls - // and normally scheduled ones. - Preferences.set("services.sync.lastTabFetch", Math.floor(Date.now() / 1000)); - Services.obs.notifyObservers(null, TOPIC_TABS_CHANGED, null); - break; - case "weave:service:start-over": - // start-over needs to notify so consumers find no tabs. - Preferences.reset("services.sync.lastTabFetch"); - Services.obs.notifyObservers(null, TOPIC_TABS_CHANGED, null); - break; - case "nsPref:changed": - Services.obs.notifyObservers(null, TOPIC_TABS_CHANGED, null); - break; - default: - break; - } - }, - - // Returns true if Sync is configured to Sync tabs, false otherwise - get isConfiguredToSyncTabs() { - if (!weaveXPCService.ready) { - log.debug("Sync isn't yet ready; assuming tab engine is enabled"); - return true; - } - - let engine = Weave.Service.engineManager.get("tabs"); - return engine && engine.enabled; - }, - - get hasSyncedThisSession() { - let engine = Weave.Service.engineManager.get("tabs"); - return engine && engine.hasSyncedThisSession; - }, -}; - -Services.obs.addObserver(SyncedTabsInternal, "weave:engine:sync:finish", false); -Services.obs.addObserver(SyncedTabsInternal, "weave:service:start-over", false); -// Observe the pref the indicates the state of the tabs engine has changed. -// This will force consumers to re-evaluate the state of sync and update -// accordingly. -Services.prefs.addObserver("services.sync.engine.tabs", SyncedTabsInternal, false); - -// The public interface. -this.SyncedTabs = { - // A mock-point for tests. - _internal: SyncedTabsInternal, - - // We make the topic for the observer notification public. - TOPIC_TABS_CHANGED, - - // Returns true if Sync is configured to Sync tabs, false otherwise - get isConfiguredToSyncTabs() { - return this._internal.isConfiguredToSyncTabs; - }, - - // Returns true if a tab sync has completed once this session. If this - // returns false, then getting back no clients/tabs possibly just means we - // are waiting for that first sync to complete. - get hasSyncedThisSession() { - return this._internal.hasSyncedThisSession; - }, - - // Return a promise that resolves with an array of client records, each with - // a .tabs array. Note that part of the contract for this module is that the - // returned objects are not shared between invocations, so callers are free - // to mutate the returned objects (eg, sort, truncate) however they see fit. - getTabClients(query) { - return this._internal.getTabClients(query); - }, - - // Starts a background request to start syncing tabs. Returns a promise that - // resolves when the sync is complete, but there's no resolved value - - // callers should be listening for TOPIC_TABS_CHANGED. - // If |force| is true we always sync. If false, we only sync if the most - // recent sync wasn't "recently". - syncTabs(force) { - return this._internal.syncTabs(force); - }, - - sortTabClientsByLastUsed(clients, maxTabs = Infinity) { - // First sort and filter the list of tabs for each client. Note that - // this module promises that the objects it returns are never - // shared, so we are free to mutate those objects directly. - for (let client of clients) { - let tabs = client.tabs; - tabs.sort((a, b) => b.lastUsed - a.lastUsed); - if (Number.isFinite(maxTabs)) { - client.tabs = tabs.slice(0, maxTabs); - } - } - // Now sort the clients - the clients are sorted in the order of the - // most recent tab for that client (ie, it is important the tabs for - // each client are already sorted.) - clients.sort((a, b) => { - if (a.tabs.length == 0) { - return 1; // b comes first. - } - if (b.tabs.length == 0) { - return -1; // a comes first. - } - return b.tabs[0].lastUsed - a.tabs[0].lastUsed; - }); - }, -}; - diff --git a/services/sync/modules/addonsreconciler.js b/services/sync/modules/addonsreconciler.js index a60fc8d56..2e838e885 100644 --- a/services/sync/modules/addonsreconciler.js +++ b/services/sync/modules/addonsreconciler.js @@ -17,7 +17,7 @@ "use strict"; -var Cu = Components.utils; +const Cu = Components.utils; Cu.import("resource://gre/modules/Log.jsm"); Cu.import("resource://services-sync/util.js"); @@ -218,12 +218,11 @@ AddonsReconciler.prototype = { } this._addons = json.addons; - for (let id in this._addons) { - let record = this._addons[id]; + for each (let record in this._addons) { record.modified = new Date(record.modified); } - for (let [time, change, id] of json.changes) { + for each (let [time, change, id] in json.changes) { this._changes.push([new Date(time), change, id]); } @@ -247,9 +246,9 @@ AddonsReconciler.prototype = { let file = path || DEFAULT_STATE_FILE; let state = {version: 1, addons: {}, changes: []}; - for (let [id, record] of Object.entries(this._addons)) { + for (let [id, record] in Iterator(this._addons)) { state.addons[id] = {}; - for (let [k, v] of Object.entries(record)) { + for (let [k, v] in Iterator(record)) { if (k == "modified") { state.addons[id][k] = v.getTime(); } @@ -259,7 +258,7 @@ AddonsReconciler.prototype = { } } - for (let [time, change, id] of this._changes) { + for each (let [time, change, id] in this._changes) { state.changes.push([time.getTime(), change, id]); } @@ -351,14 +350,14 @@ AddonsReconciler.prototype = { AddonManager.getAllAddons(function (addons) { let ids = {}; - for (let addon of addons) { + for each (let addon in addons) { ids[addon.id] = true; this.rectifyStateFromAddon(addon); } // Look for locally-defined add-ons that no longer exist and update their // record. - for (let [id, addon] of Object.entries(this._addons)) { + for (let [id, addon] in Iterator(this._addons)) { if (id in ids) { continue; } @@ -374,7 +373,7 @@ AddonsReconciler.prototype = { } let installFound = false; - for (let install of installs) { + for each (let install in installs) { if (install.addon && install.addon.id == id && install.state == AddonManager.STATE_INSTALLED) { @@ -417,7 +416,7 @@ AddonsReconciler.prototype = { * Addon instance being updated. */ rectifyStateFromAddon: function rectifyStateFromAddon(addon) { - this._log.debug(`Rectifying state for addon ${addon.name} (version=${addon.version}, id=${addon.id})`); + this._log.debug("Rectifying state for addon: " + addon.id); this._ensureStateLoaded(); let id = addon.id; @@ -434,8 +433,7 @@ AddonsReconciler.prototype = { modified: now, type: addon.type, scope: addon.scope, - foreignInstall: addon.foreignInstall, - isSyncable: addon.isSyncable, + foreignInstall: addon.foreignInstall }; this._addons[id] = record; this._log.debug("Adding change because add-on not present locally: " + @@ -445,7 +443,6 @@ AddonsReconciler.prototype = { } let record = this._addons[id]; - record.isSyncable = addon.isSyncable; if (!record.installed) { // It is possible the record is marked as uninstalled because an @@ -486,11 +483,12 @@ AddonsReconciler.prototype = { this._log.info("Change recorded for " + state.id); this._changes.push([date, change, state.id]); - for (let listener of this._listeners) { + for each (let listener in this._listeners) { try { listener.changeListener.call(listener, date, change, state); } catch (ex) { - this._log.warn("Exception calling change listener", ex); + this._log.warn("Exception calling change listener: " + + Utils.exceptionStr(ex)); } } }, @@ -556,8 +554,7 @@ AddonsReconciler.prototype = { * @return Object on success on null on failure. */ getAddonStateFromSyncGUID: function getAddonStateFromSyncGUID(guid) { - for (let id in this.addons) { - let addon = this.addons[id]; + for each (let addon in this.addons) { if (addon.guid == guid) { return addon; } @@ -636,7 +633,7 @@ AddonsReconciler.prototype = { } } catch (ex) { - this._log.warn("Exception", ex); + this._log.warn("Exception: " + Utils.exceptionStr(ex)); } }, diff --git a/services/sync/modules/addonutils.js b/services/sync/modules/addonutils.js index 95da6be0a..54b441b9e 100644 --- a/services/sync/modules/addonutils.js +++ b/services/sync/modules/addonutils.js @@ -6,7 +6,7 @@ this.EXPORTED_SYMBOLS = ["AddonUtils"]; -var {interfaces: Ci, utils: Cu} = Components; +const {interfaces: Ci, utils: Cu} = Components; Cu.import("resource://gre/modules/XPCOMUtils.jsm"); Cu.import("resource://gre/modules/Log.jsm"); @@ -38,10 +38,21 @@ AddonUtilsInternal.prototype = { * Function to be called with result of operation. */ getInstallFromSearchResult: - function getInstallFromSearchResult(addon, cb) { + function getInstallFromSearchResult(addon, cb, requireSecureURI=true) { this._log.debug("Obtaining install for " + addon.id); + // Verify that the source URI uses TLS. We don't allow installs from + // insecure sources for security reasons. The Addon Manager ensures that + // cert validation, etc is performed. + if (requireSecureURI) { + let scheme = addon.sourceURI.scheme; + if (scheme != "https") { + cb(new Error("Insecure source URI scheme: " + scheme), addon.install); + return; + } + } + // We should theoretically be able to obtain (and use) addon.install if // it is available. However, the addon.sourceURI rewriting won't be // reflected in the AddonInstall, so we can't use it. If we ever get rid @@ -69,6 +80,8 @@ AddonUtilsInternal.prototype = { * syncGUID - Sync GUID to use for the new add-on. * enabled - Boolean indicating whether the add-on should be enabled upon * install. + * requireSecureURI - Boolean indicating whether to require a secure + * URI to install from. This defaults to true. * * When complete it calls a callback with 2 arguments, error and result. * @@ -92,6 +105,10 @@ AddonUtilsInternal.prototype = { function installAddonFromSearchResult(addon, options, cb) { this._log.info("Trying to install add-on from search result: " + addon.id); + if (options.requireSecureURI === undefined) { + options.requireSecureURI = true; + } + this.getInstallFromSearchResult(addon, function onResult(error, install) { if (error) { cb(error, null); @@ -147,10 +164,10 @@ AddonUtilsInternal.prototype = { install.install(); } catch (ex) { - this._log.error("Error installing add-on", ex); + this._log.error("Error installing add-on: " + Utils.exceptionstr(ex)); cb(ex, null); } - }.bind(this)); + }.bind(this), options.requireSecureURI); }, /** @@ -231,7 +248,7 @@ AddonUtilsInternal.prototype = { } let ids = []; - for (let addon of installs) { + for each (let addon in installs) { ids.push(addon.id); } @@ -244,7 +261,6 @@ AddonUtilsInternal.prototype = { installedIDs: [], installs: [], addons: [], - skipped: [], errors: [] }; @@ -282,21 +298,15 @@ AddonUtilsInternal.prototype = { // server-side metrics aren't skewed (bug 708134). The server should // ideally send proper URLs, but this solution was deemed too // complicated at the time the functionality was implemented. - for (let addon of addons) { - // Find the specified options for this addon. - let options; - for (let install of installs) { - if (install.id == addon.id) { - options = install; - break; - } - } - if (!this.canInstallAddon(addon, options)) { - ourResult.skipped.push(addon.id); + for each (let addon in addons) { + // sourceURI presence isn't enforced by AddonRepository. So, we skip + // add-ons without a sourceURI. + if (!addon.sourceURI) { + this._log.info("Skipping install of add-on because missing " + + "sourceURI: " + addon.id); continue; } - // We can go ahead and attempt to install it. toInstall.push(addon); // We should always be able to QI the nsIURI to nsIURL. If not, we @@ -332,9 +342,9 @@ AddonUtilsInternal.prototype = { // Start all the installs asynchronously. They will report back to us // as they finish, eventually triggering the global callback. - for (let addon of toInstall) { + for each (let addon in toInstall) { let options = {}; - for (let install of installs) { + for each (let install in installs) { if (install.id == addon.id) { options = install; break; @@ -353,51 +363,9 @@ AddonUtilsInternal.prototype = { }, /** - * Returns true if we are able to install the specified addon, false - * otherwise. It is expected that this will log the reason if it returns - * false. - * - * @param addon - * (Addon) Add-on instance to check. - * @param options - * (object) The options specified for this addon. See installAddons() - * for the valid elements. - */ - canInstallAddon(addon, options) { - // sourceURI presence isn't enforced by AddonRepository. So, we skip - // add-ons without a sourceURI. - if (!addon.sourceURI) { - this._log.info("Skipping install of add-on because missing " + - "sourceURI: " + addon.id); - return false; - } - // Verify that the source URI uses TLS. We don't allow installs from - // insecure sources for security reasons. The Addon Manager ensures - // that cert validation etc is performed. - // (We should also consider just dropping this entirely and calling - // XPIProvider.isInstallAllowed, but that has additional semantics we might - // need to think through...) - let requireSecureURI = true; - if (options && options.requireSecureURI !== undefined) { - requireSecureURI = options.requireSecureURI; - } - - if (requireSecureURI) { - let scheme = addon.sourceURI.scheme; - if (scheme != "https") { - this._log.info(`Skipping install of add-on "${addon.id}" because sourceURI's scheme of "${scheme}" is not trusted`); - return false; - } - } - this._log.info(`Add-on "${addon.id}" is able to be installed`); - return true; - }, - - - /** * Update the user disabled flag for an add-on. * - * The supplied callback will be called when the operation is + * The supplied callback will ba called when the operation is * complete. If the new flag matches the existing or if the add-on * isn't currently active, the function will fire the callback * immediately. Else, the callback is invoked when the AddonManager diff --git a/services/sync/modules/bookmark_validator.js b/services/sync/modules/bookmark_validator.js deleted file mode 100644 index 2a94ba043..000000000 --- a/services/sync/modules/bookmark_validator.js +++ /dev/null @@ -1,784 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this file, - * You can obtain one at http://mozilla.org/MPL/2.0/. */ - -"use strict"; - -const Cu = Components.utils; - -Cu.import("resource://gre/modules/PlacesUtils.jsm"); -Cu.import("resource://gre/modules/PlacesSyncUtils.jsm"); -Cu.import("resource://gre/modules/Task.jsm"); -Cu.import("resource://gre/modules/XPCOMUtils.jsm"); - - -this.EXPORTED_SYMBOLS = ["BookmarkValidator", "BookmarkProblemData"]; - -const LEFT_PANE_ROOT_ANNO = "PlacesOrganizer/OrganizerFolder"; -const LEFT_PANE_QUERY_ANNO = "PlacesOrganizer/OrganizerQuery"; - -// Indicates if a local bookmark tree node should be excluded from syncing. -function isNodeIgnored(treeNode) { - return treeNode.annos && treeNode.annos.some(anno => anno.name == LEFT_PANE_ROOT_ANNO || - anno.name == LEFT_PANE_QUERY_ANNO); -} -const BOOKMARK_VALIDATOR_VERSION = 1; - -/** - * Result of bookmark validation. Contains the following fields which describe - * server-side problems unless otherwise specified. - * - * - missingIDs (number): # of objects with missing ids - * - duplicates (array of ids): ids seen more than once - * - parentChildMismatches (array of {parent: parentid, child: childid}): - * instances where the child's parentid and the parent's children array - * do not match - * - cycles (array of array of ids). List of cycles found in the server-side tree. - * - clientCycles (array of array of ids). List of cycles found in the client-side tree. - * - orphans (array of {id: string, parent: string}): List of nodes with - * either no parentid, or where the parent could not be found. - * - missingChildren (array of {parent: id, child: id}): - * List of parent/children where the child id couldn't be found - * - deletedChildren (array of { parent: id, child: id }): - * List of parent/children where child id was a deleted item (but still showed up - * in the children array) - * - multipleParents (array of {child: id, parents: array of ids}): - * List of children that were part of multiple parent arrays - * - deletedParents (array of ids) : List of records that aren't deleted but - * had deleted parents - * - childrenOnNonFolder (array of ids): list of non-folders that still have - * children arrays - * - duplicateChildren (array of ids): list of records who have the same - * child listed multiple times in their children array - * - parentNotFolder (array of ids): list of records that have parents that - * aren't folders - * - rootOnServer (boolean): true if the root came from the server - * - badClientRoots (array of ids): Contains any client-side root ids where - * the root is missing or isn't a (direct) child of the places root. - * - * - clientMissing: Array of ids on the server missing from the client - * - serverMissing: Array of ids on the client missing from the server - * - serverDeleted: Array of ids on the client that the server had marked as deleted. - * - serverUnexpected: Array of ids that appear on the server but shouldn't - * because the client attempts to never upload them. - * - differences: Array of {id: string, differences: string array} recording - * the non-structural properties that are differente between the client and server - * - structuralDifferences: As above, but contains the items where the differences were - * structural, that is, they contained childGUIDs or parentid - */ -class BookmarkProblemData { - constructor() { - this.rootOnServer = false; - this.missingIDs = 0; - - this.duplicates = []; - this.parentChildMismatches = []; - this.cycles = []; - this.clientCycles = []; - this.orphans = []; - this.missingChildren = []; - this.deletedChildren = []; - this.multipleParents = []; - this.deletedParents = []; - this.childrenOnNonFolder = []; - this.duplicateChildren = []; - this.parentNotFolder = []; - - this.badClientRoots = []; - this.clientMissing = []; - this.serverMissing = []; - this.serverDeleted = []; - this.serverUnexpected = []; - this.differences = []; - this.structuralDifferences = []; - } - - /** - * Convert ("difference", [{ differences: ["tags", "name"] }, { differences: ["name"] }]) into - * [{ name: "difference:tags", count: 1}, { name: "difference:name", count: 2 }], etc. - */ - _summarizeDifferences(prefix, diffs) { - let diffCounts = new Map(); - for (let { differences } of diffs) { - for (let type of differences) { - let name = prefix + ":" + type; - let count = diffCounts.get(name) || 0; - diffCounts.set(name, count + 1); - } - } - return [...diffCounts].map(([name, count]) => ({ name, count })); - } - - /** - * Produce a list summarizing problems found. Each entry contains {name, count}, - * where name is the field name for the problem, and count is the number of times - * the problem was encountered. - * - * Validation has failed if all counts are not 0. - * - * If the `full` argument is truthy, we also include information about which - * properties we saw structural differences in. Currently, this means either - * "sdiff:parentid" and "sdiff:childGUIDS" may be present. - */ - getSummary(full) { - let result = [ - { name: "clientMissing", count: this.clientMissing.length }, - { name: "serverMissing", count: this.serverMissing.length }, - { name: "serverDeleted", count: this.serverDeleted.length }, - { name: "serverUnexpected", count: this.serverUnexpected.length }, - - { name: "structuralDifferences", count: this.structuralDifferences.length }, - { name: "differences", count: this.differences.length }, - - { name: "missingIDs", count: this.missingIDs }, - { name: "rootOnServer", count: this.rootOnServer ? 1 : 0 }, - - { name: "duplicates", count: this.duplicates.length }, - { name: "parentChildMismatches", count: this.parentChildMismatches.length }, - { name: "cycles", count: this.cycles.length }, - { name: "clientCycles", count: this.clientCycles.length }, - { name: "badClientRoots", count: this.badClientRoots.length }, - { name: "orphans", count: this.orphans.length }, - { name: "missingChildren", count: this.missingChildren.length }, - { name: "deletedChildren", count: this.deletedChildren.length }, - { name: "multipleParents", count: this.multipleParents.length }, - { name: "deletedParents", count: this.deletedParents.length }, - { name: "childrenOnNonFolder", count: this.childrenOnNonFolder.length }, - { name: "duplicateChildren", count: this.duplicateChildren.length }, - { name: "parentNotFolder", count: this.parentNotFolder.length }, - ]; - if (full) { - let structural = this._summarizeDifferences("sdiff", this.structuralDifferences); - result.push.apply(result, structural); - } - return result; - } -} - -// Defined lazily to avoid initializing PlacesUtils.bookmarks too soon. -XPCOMUtils.defineLazyGetter(this, "SYNCED_ROOTS", () => [ - PlacesUtils.bookmarks.menuGuid, - PlacesUtils.bookmarks.toolbarGuid, - PlacesUtils.bookmarks.unfiledGuid, - PlacesUtils.bookmarks.mobileGuid, -]); - -class BookmarkValidator { - - _followQueries(recordMap) { - for (let [guid, entry] of recordMap) { - if (entry.type !== "query" && (!entry.bmkUri || !entry.bmkUri.startsWith("place:"))) { - continue; - } - // Might be worth trying to parse the place: query instead so that this - // works "automatically" with things like aboutsync. - let queryNodeParent = PlacesUtils.getFolderContents(entry, false, true); - if (!queryNodeParent || !queryNodeParent.root.hasChildren) { - continue; - } - queryNodeParent = queryNodeParent.root; - let queryNode = null; - let numSiblings = 0; - let containerWasOpen = queryNodeParent.containerOpen; - queryNodeParent.containerOpen = true; - try { - try { - numSiblings = queryNodeParent.childCount; - } catch (e) { - // This throws when we can't actually get the children. This is the - // case for history containers, tag queries, ... - continue; - } - for (let i = 0; i < numSiblings && !queryNode; ++i) { - let child = queryNodeParent.getChild(i); - if (child && child.bookmarkGuid && child.bookmarkGuid === guid) { - queryNode = child; - } - } - } finally { - queryNodeParent.containerOpen = containerWasOpen; - } - if (!queryNode) { - continue; - } - - let concreteId = PlacesUtils.getConcreteItemGuid(queryNode); - if (!concreteId) { - continue; - } - let concreteItem = recordMap.get(concreteId); - if (!concreteItem) { - continue; - } - entry.concrete = concreteItem; - } - } - - createClientRecordsFromTree(clientTree) { - // Iterate over the treeNode, converting it to something more similar to what - // the server stores. - let records = []; - let recordsByGuid = new Map(); - let syncedRoots = SYNCED_ROOTS; - function traverse(treeNode, synced) { - if (!synced) { - synced = syncedRoots.includes(treeNode.guid); - } else if (isNodeIgnored(treeNode)) { - synced = false; - } - let guid = PlacesSyncUtils.bookmarks.guidToSyncId(treeNode.guid); - let itemType = 'item'; - treeNode.ignored = !synced; - treeNode.id = guid; - switch (treeNode.type) { - case PlacesUtils.TYPE_X_MOZ_PLACE: - let query = null; - if (treeNode.annos && treeNode.uri.startsWith("place:")) { - query = treeNode.annos.find(({name}) => - name === PlacesSyncUtils.bookmarks.SMART_BOOKMARKS_ANNO); - } - if (query && query.value) { - itemType = 'query'; - } else { - itemType = 'bookmark'; - } - break; - case PlacesUtils.TYPE_X_MOZ_PLACE_CONTAINER: - let isLivemark = false; - if (treeNode.annos) { - for (let anno of treeNode.annos) { - if (anno.name === PlacesUtils.LMANNO_FEEDURI) { - isLivemark = true; - treeNode.feedUri = anno.value; - } else if (anno.name === PlacesUtils.LMANNO_SITEURI) { - isLivemark = true; - treeNode.siteUri = anno.value; - } - } - } - itemType = isLivemark ? "livemark" : "folder"; - break; - case PlacesUtils.TYPE_X_MOZ_PLACE_SEPARATOR: - itemType = 'separator'; - break; - } - - if (treeNode.tags) { - treeNode.tags = treeNode.tags.split(","); - } else { - treeNode.tags = []; - } - treeNode.type = itemType; - treeNode.pos = treeNode.index; - treeNode.bmkUri = treeNode.uri; - records.push(treeNode); - // We want to use the "real" guid here. - recordsByGuid.set(treeNode.guid, treeNode); - if (treeNode.type === 'folder') { - treeNode.childGUIDs = []; - if (!treeNode.children) { - treeNode.children = []; - } - for (let child of treeNode.children) { - traverse(child, synced); - child.parent = treeNode; - child.parentid = guid; - treeNode.childGUIDs.push(child.guid); - } - } - } - traverse(clientTree, false); - clientTree.id = 'places'; - this._followQueries(recordsByGuid); - return records; - } - - /** - * Process the server-side list. Mainly this builds the records into a tree, - * but it also records information about problems, and produces arrays of the - * deleted and non-deleted nodes. - * - * Returns an object containing: - * - records:Array of non-deleted records. Each record contains the following - * properties - * - childGUIDs (array of strings, only present if type is 'folder'): the - * list of child GUIDs stored on the server. - * - children (array of records, only present if type is 'folder'): - * each record has these same properties. This may differ in content - * from what you may expect from the childGUIDs list, as it won't - * contain any records that could not be found. - * - parent (record): The parent to this record. - * - Unchanged properties send down from the server: id, title, type, - * parentName, parentid, bmkURI, keyword, tags, pos, queryId, loadInSidebar - * - root: Root of the server-side bookmark tree. Has the same properties as - * above. - * - deletedRecords: As above, but only contains items that the server sent - * where it also sent indication that the item should be deleted. - * - problemData: a BookmarkProblemData object, with the caveat that - * the fields describing client/server relationship will not have been filled - * out yet. - */ - inspectServerRecords(serverRecords) { - let deletedItemIds = new Set(); - let idToRecord = new Map(); - let deletedRecords = []; - - let folders = []; - let problems = []; - - let problemData = new BookmarkProblemData(); - - let resultRecords = []; - - for (let record of serverRecords) { - if (!record.id) { - ++problemData.missingIDs; - continue; - } - if (record.deleted) { - deletedItemIds.add(record.id); - } else { - if (idToRecord.has(record.id)) { - problemData.duplicates.push(record.id); - continue; - } - } - idToRecord.set(record.id, record); - - if (record.children) { - if (record.type !== "folder") { - // Due to implementation details in engines/bookmarks.js, (Livemark - // subclassing BookmarkFolder) Livemarks will have a children array, - // but it should still be empty. - if (!record.children.length) { - continue; - } - // Otherwise we mark it as an error and still try to resolve the children - problemData.childrenOnNonFolder.push(record.id); - } - folders.push(record); - - if (new Set(record.children).size !== record.children.length) { - problemData.duplicateChildren.push(record.id) - } - - // The children array stores special guids as their local guid values, - // e.g. 'menu________' instead of 'menu', but all other parts of the - // serverside bookmark info stores it as the special value ('menu'). - record.childGUIDs = record.children; - record.children = record.children.map(childID => { - return PlacesSyncUtils.bookmarks.guidToSyncId(childID); - }); - } - } - - for (let deletedId of deletedItemIds) { - let record = idToRecord.get(deletedId); - if (record && !record.isDeleted) { - deletedRecords.push(record); - record.isDeleted = true; - } - } - - let root = idToRecord.get('places'); - - if (!root) { - // Fabricate a root. We want to remember that it's fake so that we can - // avoid complaining about stuff like it missing it's childGUIDs later. - root = { id: 'places', children: [], type: 'folder', title: '', fake: true }; - resultRecords.push(root); - idToRecord.set('places', root); - } else { - problemData.rootOnServer = true; - } - - // Build the tree, find orphans, and record most problems having to do with - // the tree structure. - for (let [id, record] of idToRecord) { - if (record === root) { - continue; - } - - if (record.isDeleted) { - continue; - } - - let parentID = record.parentid; - if (!parentID) { - problemData.orphans.push({id: record.id, parent: parentID}); - continue; - } - - let parent = idToRecord.get(parentID); - if (!parent) { - problemData.orphans.push({id: record.id, parent: parentID}); - continue; - } - - if (parent.type !== 'folder') { - problemData.parentNotFolder.push(record.id); - if (!parent.children) { - parent.children = []; - } - if (!parent.childGUIDs) { - parent.childGUIDs = []; - } - } - - if (!record.isDeleted) { - resultRecords.push(record); - } - - record.parent = parent; - if (parent !== root || problemData.rootOnServer) { - let childIndex = parent.children.indexOf(id); - if (childIndex < 0) { - problemData.parentChildMismatches.push({parent: parent.id, child: record.id}); - } else { - parent.children[childIndex] = record; - } - } else { - parent.children.push(record); - } - - if (parent.isDeleted && !record.isDeleted) { - problemData.deletedParents.push(record.id); - } - - // We used to check if the parentName on the server matches the actual - // local parent name, but given this is used only for de-duping a record - // the first time it is seen and expensive to keep up-to-date, we decided - // to just stop recording it. See bug 1276969 for more. - } - - // Check that we aren't missing any children. - for (let folder of folders) { - folder.unfilteredChildren = folder.children; - folder.children = []; - for (let ci = 0; ci < folder.unfilteredChildren.length; ++ci) { - let child = folder.unfilteredChildren[ci]; - let childObject; - if (typeof child == "string") { - // This can happen the parent refers to a child that has a different - // parentid, or if it refers to a missing or deleted child. It shouldn't - // be possible with totally valid bookmarks. - childObject = idToRecord.get(child); - if (!childObject) { - problemData.missingChildren.push({parent: folder.id, child}); - } else { - folder.unfilteredChildren[ci] = childObject; - if (childObject.isDeleted) { - problemData.deletedChildren.push({ parent: folder.id, child }); - } - } - } else { - childObject = child; - } - - if (!childObject) { - continue; - } - - if (childObject.parentid === folder.id) { - folder.children.push(childObject); - continue; - } - - // The child is very probably in multiple `children` arrays -- - // see if we already have a problem record about it. - let currentProblemRecord = problemData.multipleParents.find(pr => - pr.child === child); - - if (currentProblemRecord) { - currentProblemRecord.parents.push(folder.id); - continue; - } - - let otherParent = idToRecord.get(childObject.parentid); - // it's really an ... orphan ... sort of. - if (!otherParent) { - // if we never end up adding to this parent's list, we filter it out after this loop. - problemData.multipleParents.push({ - child, - parents: [folder.id] - }); - if (!problemData.orphans.some(r => r.id === child)) { - problemData.orphans.push({ - id: child, - parent: childObject.parentid - }); - } - continue; - } - - if (otherParent.isDeleted) { - if (!problemData.deletedParents.includes(child)) { - problemData.deletedParents.push(child); - } - continue; - } - - if (otherParent.childGUIDs && !otherParent.childGUIDs.includes(child)) { - if (!problemData.parentChildMismatches.some(r => r.child === child)) { - // Might not be possible to get here. - problemData.parentChildMismatches.push({ child, parent: folder.id }); - } - } - - problemData.multipleParents.push({ - child, - parents: [childObject.parentid, folder.id] - }); - } - } - problemData.multipleParents = problemData.multipleParents.filter(record => - record.parents.length >= 2); - - problemData.cycles = this._detectCycles(resultRecords); - - return { - deletedRecords, - records: resultRecords, - problemData, - root, - }; - } - - // helper for inspectServerRecords - _detectCycles(records) { - // currentPath and pathLookup contain the same data. pathLookup is faster to - // query, but currentPath gives is the order of traversal that we need in - // order to report the members of the cycles. - let pathLookup = new Set(); - let currentPath = []; - let cycles = []; - let seenEver = new Set(); - const traverse = node => { - if (pathLookup.has(node)) { - let cycleStart = currentPath.lastIndexOf(node); - let cyclePath = currentPath.slice(cycleStart).map(n => n.id); - cycles.push(cyclePath); - return; - } else if (seenEver.has(node)) { - // If we're checking the server, this is a problem, but it should already be reported. - // On the client, this could happen due to including `node.concrete` in the child list. - return; - } - seenEver.add(node); - let children = node.children || []; - if (node.concrete) { - children.push(node.concrete); - } - if (children) { - pathLookup.add(node); - currentPath.push(node); - for (let child of children) { - traverse(child); - } - currentPath.pop(); - pathLookup.delete(node); - } - }; - for (let record of records) { - if (!seenEver.has(record)) { - traverse(record); - } - } - - return cycles; - } - - // Perform client-side sanity checking that doesn't involve server data - _validateClient(problemData, clientRecords) { - problemData.clientCycles = this._detectCycles(clientRecords); - for (let rootGUID of SYNCED_ROOTS) { - let record = clientRecords.find(record => - record.guid === rootGUID); - if (!record || record.parentid !== "places") { - problemData.badClientRoots.push(rootGUID); - } - } - } - - /** - * Compare the list of server records with the client tree. - * - * Returns the same data as described in the inspectServerRecords comment, - * with the following additional fields. - * - clientRecords: an array of client records in a similar format to - * the .records (ie, server records) entry. - * - problemData is the same as for inspectServerRecords, except all properties - * will be filled out. - */ - compareServerWithClient(serverRecords, clientTree) { - - let clientRecords = this.createClientRecordsFromTree(clientTree); - let inspectionInfo = this.inspectServerRecords(serverRecords); - inspectionInfo.clientRecords = clientRecords; - - // Mainly do this to remove deleted items and normalize child guids. - serverRecords = inspectionInfo.records; - let problemData = inspectionInfo.problemData; - - this._validateClient(problemData, clientRecords); - - let matches = []; - - let allRecords = new Map(); - let serverDeletedLookup = new Set(inspectionInfo.deletedRecords.map(r => r.id)); - - for (let sr of serverRecords) { - if (sr.fake) { - continue; - } - allRecords.set(sr.id, {client: null, server: sr}); - } - - for (let cr of clientRecords) { - let unified = allRecords.get(cr.id); - if (!unified) { - allRecords.set(cr.id, {client: cr, server: null}); - } else { - unified.client = cr; - } - } - - - for (let [id, {client, server}] of allRecords) { - if (!client && server) { - problemData.clientMissing.push(id); - continue; - } - if (!server && client) { - if (serverDeletedLookup.has(id)) { - problemData.serverDeleted.push(id); - } else if (!client.ignored && client.id != "places") { - problemData.serverMissing.push(id); - } - continue; - } - if (server && client && client.ignored) { - problemData.serverUnexpected.push(id); - } - let differences = []; - let structuralDifferences = []; - - // Don't bother comparing titles of roots. It's okay if locally it's - // "Mobile Bookmarks", but the server thinks it's "mobile". - // TODO: We probably should be handing other localized bookmarks (e.g. - // default bookmarks) here as well, see bug 1316041. - if (!SYNCED_ROOTS.includes(client.guid)) { - // We want to treat undefined, null and an empty string as identical - if ((client.title || "") !== (server.title || "")) { - differences.push("title"); - } - } - - if (client.parentid || server.parentid) { - if (client.parentid !== server.parentid) { - structuralDifferences.push('parentid'); - } - } - - if (client.tags || server.tags) { - let cl = client.tags || []; - let sl = server.tags || []; - if (cl.length !== sl.length || !cl.every((tag, i) => sl.indexOf(tag) >= 0)) { - differences.push('tags'); - } - } - - let sameType = client.type === server.type; - if (!sameType) { - if (server.type === "query" && client.type === "bookmark" && client.bmkUri.startsWith("place:")) { - sameType = true; - } - } - - - if (!sameType) { - differences.push('type'); - } else { - switch (server.type) { - case 'bookmark': - case 'query': - if (server.bmkUri !== client.bmkUri) { - differences.push('bmkUri'); - } - break; - case "livemark": - if (server.feedUri != client.feedUri) { - differences.push("feedUri"); - } - if (server.siteUri != client.siteUri) { - differences.push("siteUri"); - } - break; - case 'folder': - if (server.id === 'places' && !problemData.rootOnServer) { - // It's the fabricated places root. It won't have the GUIDs, but - // it doesn't matter. - break; - } - if (client.childGUIDs || server.childGUIDs) { - let cl = client.childGUIDs || []; - let sl = server.childGUIDs || []; - if (cl.length !== sl.length || !cl.every((id, i) => sl[i] === id)) { - structuralDifferences.push('childGUIDs'); - } - } - break; - } - } - - if (differences.length) { - problemData.differences.push({id, differences}); - } - if (structuralDifferences.length) { - problemData.structuralDifferences.push({ id, differences: structuralDifferences }); - } - } - return inspectionInfo; - } - - _getServerState(engine) { - let collection = engine.itemSource(); - let collectionKey = engine.service.collectionKeys.keyForCollection(engine.name); - collection.full = true; - let items = []; - collection.recordHandler = function(item) { - item.decrypt(collectionKey); - items.push(item.cleartext); - }; - let resp = collection.getBatched(); - if (!resp.success) { - throw resp; - } - return items; - } - - validate(engine) { - let self = this; - return Task.spawn(function*() { - let start = Date.now(); - let clientTree = yield PlacesUtils.promiseBookmarksTree("", { - includeItemIds: true - }); - let serverState = self._getServerState(engine); - let serverRecordCount = serverState.length; - let result = self.compareServerWithClient(serverState, clientTree); - let end = Date.now(); - let duration = end-start; - return { - duration, - version: self.version, - problems: result.problemData, - recordCount: serverRecordCount - }; - }); - } - -}; - -BookmarkValidator.prototype.version = BOOKMARK_VALIDATOR_VERSION; - diff --git a/services/sync/modules/browserid_identity.js b/services/sync/modules/browserid_identity.js index db3821518..bc8ea6b30 100644 --- a/services/sync/modules/browserid_identity.js +++ b/services/sync/modules/browserid_identity.js @@ -4,9 +4,9 @@ "use strict"; -this.EXPORTED_SYMBOLS = ["BrowserIDManager", "AuthenticationError"]; +this.EXPORTED_SYMBOLS = ["BrowserIDManager"]; -var {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components; +const {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components; Cu.import("resource://gre/modules/Log.jsm"); Cu.import("resource://services-common/async.js"); @@ -39,13 +39,12 @@ XPCOMUtils.defineLazyGetter(this, 'log', function() { }); // FxAccountsCommon.js doesn't use a "namespace", so create one here. -var fxAccountsCommon = {}; +let fxAccountsCommon = {}; Cu.import("resource://gre/modules/FxAccountsCommon.js", fxAccountsCommon); const OBSERVER_TOPICS = [ fxAccountsCommon.ONLOGIN_NOTIFICATION, fxAccountsCommon.ONLOGOUT_NOTIFICATION, - fxAccountsCommon.ON_ACCOUNT_STATE_CHANGE_NOTIFICATION, ]; const PREF_SYNC_SHOW_CUSTOMIZATION = "services.sync-setup.ui.showCustomizationDialog"; @@ -66,9 +65,8 @@ function deriveKeyBundle(kB) { some other error object (which should do the right thing when toString() is called on it) */ -function AuthenticationError(details, source) { +function AuthenticationError(details) { this.details = details; - this.source = source; } AuthenticationError.prototype = { @@ -106,6 +104,12 @@ this.BrowserIDManager.prototype = { // we don't consider the lack of a keybundle as a failure state. _shouldHaveSyncKeyBundle: false, + get readyToAuthenticate() { + // We are finished initializing when we *should* have a sync key bundle, + // although we might not actually have one due to auth failures etc. + return this._shouldHaveSyncKeyBundle; + }, + get needsCustomization() { try { return Services.prefs.getBoolPref(PREF_SYNC_SHOW_CUSTOMIZATION); @@ -114,34 +118,11 @@ this.BrowserIDManager.prototype = { } }, - hashedUID() { - if (!this._token) { - throw new Error("hashedUID: Don't have token"); - } - return this._token.hashed_fxa_uid - }, - - deviceID() { - return this._signedInUser && this._signedInUser.deviceId; - }, - initialize: function() { for (let topic of OBSERVER_TOPICS) { Services.obs.addObserver(this, topic, false); } - // and a background fetch of account data just so we can set this.account, - // so we have a username available before we've actually done a login. - // XXX - this is actually a hack just for tests and really shouldn't be - // necessary. Also, you'd think it would be safe to allow this.account to - // be set to null when there's no user logged in, but argue with the test - // suite, not with me :) - this._fxaService.getSignedInUser().then(accountData => { - if (accountData) { - this.account = accountData.email; - } - }).catch(err => { - // As above, this is only for tests so it is safe to ignore. - }); + return this.initializeWithCurrentIdentity(); }, /** @@ -149,7 +130,7 @@ this.BrowserIDManager.prototype = { * the user is logged in, or is rejected if the login attempt has failed. */ ensureLoggedIn: function() { - if (!this._shouldHaveSyncKeyBundle && this.whenReadyToAuthenticate) { + if (!this._shouldHaveSyncKeyBundle) { // We are already in the process of logging in. return this.whenReadyToAuthenticate.promise; } @@ -163,7 +144,7 @@ this.BrowserIDManager.prototype = { // re-entering of credentials by the user is necessary we don't take any // further action - an observer will fire when the user does that. if (Weave.Status.login == LOGIN_FAILED_LOGIN_REJECTED) { - return Promise.reject(new Error("User needs to re-authenticate")); + return Promise.reject(); } // So - we've a previous auth problem and aren't currently attempting to @@ -179,6 +160,7 @@ this.BrowserIDManager.prototype = { } this.resetCredentials(); this._signedInUser = null; + return Promise.resolve(); }, offerSyncOptions: function () { @@ -202,7 +184,7 @@ this.BrowserIDManager.prototype = { // Reset the world before we do anything async. this.whenReadyToAuthenticate = Promise.defer(); - this.whenReadyToAuthenticate.promise.catch(err => { + this.whenReadyToAuthenticate.promise.then(null, (err) => { this._log.error("Could not authenticate", err); }); @@ -258,14 +240,14 @@ this.BrowserIDManager.prototype = { Services.obs.notifyObservers(null, "weave:service:setup-complete", null); Weave.Utils.nextTick(Weave.Service.sync, Weave.Service); } - }).catch(authErr => { - // report what failed... - this._log.error("Background fetch for key bundle failed", authErr); + }).then(null, err => { this._shouldHaveSyncKeyBundle = true; // but we probably don't have one... - this.whenReadyToAuthenticate.reject(authErr); + this.whenReadyToAuthenticate.reject(err); + // report what failed... + this._log.error("Background fetch for key bundle failed", err); }); // and we are done - the fetch continues on in the background... - }).catch(err => { + }).then(null, err => { this._log.error("Processing logged in account", err); }); }, @@ -301,8 +283,7 @@ this.BrowserIDManager.prototype = { // reauth with the server - in that case we will also get here, but // should have the same identity. // initializeWithCurrentIdentity will throw and log if these constraints - // aren't met (indirectly, via _updateSignedInUser()), so just go ahead - // and do the init. + // aren't met, so just go ahead and do the init. this.initializeWithCurrentIdentity(true); break; @@ -311,13 +292,6 @@ this.BrowserIDManager.prototype = { // startOver will cause this instance to be thrown away, so there's // nothing else to do. break; - - case fxAccountsCommon.ON_ACCOUNT_STATE_CHANGE_NOTIFICATION: - // throw away token and fetch a new one - this.resetCredentials(); - this._ensureValidToken().catch(err => - this._log.error("Error while fetching a new token", err)); - break; } }, @@ -413,9 +387,6 @@ this.BrowserIDManager.prototype = { resetCredentials: function() { this.resetSyncKey(); this._token = null; - // The cluster URL comes from the token, so resetting it to empty will - // force Sync to not accidentally use a value from an earlier token. - Weave.Service.clusterURL = null; }, /** @@ -503,12 +474,7 @@ this.BrowserIDManager.prototype = { // If we still can't get keys it probably means the user authenticated // without unlocking the MP or cleared the saved logins, so we've now // lost them - the user will need to reauth before continuing. - let result; - if (this._canFetchKeys()) { - result = STATUS_OK; - } else { - result = LOGIN_FAILED_LOGIN_REJECTED; - } + let result = this._canFetchKeys() ? STATUS_OK : LOGIN_FAILED_LOGIN_REJECTED; log.debug("unlockAndVerifyAuthState re-fetched credentials and is returning", result); return result; } @@ -540,27 +506,14 @@ this.BrowserIDManager.prototype = { return true; }, - // Get our tokenServerURL - a private helper. Returns a string. - get _tokenServerUrl() { - // We used to support services.sync.tokenServerURI but this was a - // pain-point for people using non-default servers as Sync may auto-reset - // all services.sync prefs. So if that still exists, it wins. - let url = Svc.Prefs.get("tokenServerURI"); // Svc.Prefs "root" is services.sync - if (!url) { - url = Services.prefs.getCharPref("identity.sync.tokenserver.uri"); - } - while (url.endsWith("/")) { // trailing slashes cause problems... - url = url.slice(0, -1); - } - return url; - }, - // Refresh the sync token for our user. Returns a promise that resolves // with a token (which may be null in one sad edge-case), or rejects with an // error. _fetchTokenForUser: function() { - // tokenServerURI is mis-named - convention is uri means nsISomething... - let tokenServerURI = this._tokenServerUrl; + let tokenServerURI = Svc.Prefs.get("tokenServerURI"); + if (tokenServerURI.endsWith("/")) { // trailing slashes cause problems... + tokenServerURI = tokenServerURI.slice(0, -1); + } let log = this._log; let client = this._tokenServerClient; let fxa = this._fxaService; @@ -589,7 +542,7 @@ this.BrowserIDManager.prototype = { ); } - let getToken = assertion => { + let getToken = (tokenServerURI, assertion) => { log.debug("Getting a token"); let deferred = Promise.defer(); let cb = function (err, token) { @@ -617,18 +570,7 @@ this.BrowserIDManager.prototype = { return fxa.whenVerified(this._signedInUser) .then(() => maybeFetchKeys()) .then(() => getAssertion()) - .then(assertion => getToken(assertion)) - .catch(err => { - // If we get a 401 fetching the token it may be that our certificate - // needs to be regenerated. - if (!err.response || err.response.status !== 401) { - return Promise.reject(err); - } - log.warn("Token server returned 401, refreshing certificate and retrying token fetch"); - return fxa.invalidateCertificate() - .then(() => getAssertion()) - .then(assertion => getToken(assertion)) - }) + .then(assertion => getToken(tokenServerURI, assertion)) .then(token => { // TODO: Make it be only 80% of the duration, so refresh the token // before it actually expires. This is to avoid sync storage errors @@ -640,18 +582,15 @@ this.BrowserIDManager.prototype = { } return token; }) - .catch(err => { + .then(null, err => { // TODO: unify these errors - we need to handle errors thrown by // both tokenserverclient and hawkclient. // A tokenserver error thrown based on a bad response. if (err.response && err.response.status === 401) { - err = new AuthenticationError(err, "tokenserver"); + err = new AuthenticationError(err); // A hawkclient error. } else if (err.code && err.code === 401) { - err = new AuthenticationError(err, "hawkclient"); - // An FxAccounts.jsm error. - } else if (err.message == fxAccountsCommon.ERROR_AUTH_ERROR) { - err = new AuthenticationError(err, "fxaccounts"); + err = new AuthenticationError(err); } // TODO: write tests to make sure that different auth error cases are handled here @@ -673,6 +612,7 @@ this.BrowserIDManager.prototype = { // that there is no authentication dance still under way. this._shouldHaveSyncKeyBundle = true; Weave.Status.login = this._authFailureReason; + Services.obs.notifyObservers(null, "weave:ui:login:error", null); throw err; }); }, @@ -684,19 +624,12 @@ this.BrowserIDManager.prototype = { this._log.debug("_ensureValidToken already has one"); return Promise.resolve(); } - const notifyStateChanged = - () => Services.obs.notifyObservers(null, "weave:service:login:change", null); // reset this._token as a safety net to reduce the possibility of us // repeatedly attempting to use an invalid token if _fetchTokenForUser throws. this._token = null; return this._fetchTokenForUser().then( token => { this._token = token; - notifyStateChanged(); - }, - error => { - notifyStateChanged(); - throw error } ); }, @@ -719,16 +652,9 @@ this.BrowserIDManager.prototype = { _getAuthenticationHeader: function(httpObject, method) { let cb = Async.makeSpinningCallback(); this._ensureValidToken().then(cb, cb); - // Note that in failure states we return null, causing the request to be - // made without authorization headers, thereby presumably causing a 401, - // which causes Sync to log out. If we throw, this may not happen as - // expected. try { cb.wait(); } catch (ex) { - if (Async.isShutdownException(ex)) { - throw ex; - } this._log.error("Failed to fetch a token for authentication", ex); return null; } @@ -764,17 +690,8 @@ this.BrowserIDManager.prototype = { createClusterManager: function(service) { return new BrowserIDClusterManager(service); - }, + } - // Tell Sync what the login status should be if it saw a 401 fetching - // info/collections as part of login verification (typically immediately - // after login.) - // In our case, it almost certainly means a transient error fetching a token - // (and hitting this will cause us to logout, which will correctly handle an - // authoritative login issue.) - loginStatusFromVerification404() { - return LOGIN_FAILED_NETWORK_ERROR; - }, }; /* An implementation of the ClusterManager for this identity @@ -820,7 +737,7 @@ BrowserIDClusterManager.prototype = { // it's likely a 401 was received using the existing token - in which // case we just discard the existing token and fetch a new one. if (this.service.clusterURL) { - log.debug("_findCluster has a pre-existing clusterURL, so discarding the current token"); + log.debug("_findCluster found existing clusterURL, so discarding the current token"); this.identity._token = null; } return this.identity._ensureValidToken(); diff --git a/services/sync/modules/collection_validator.js b/services/sync/modules/collection_validator.js deleted file mode 100644 index 41141bba3..000000000 --- a/services/sync/modules/collection_validator.js +++ /dev/null @@ -1,204 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this file, - * You can obtain one at http://mozilla.org/MPL/2.0/. */ - -"use strict"; - -const Cu = Components.utils; - -Cu.import("resource://services-sync/record.js"); -Cu.import("resource://services-sync/main.js"); - -this.EXPORTED_SYMBOLS = ["CollectionValidator", "CollectionProblemData"]; - -class CollectionProblemData { - constructor() { - this.missingIDs = 0; - this.duplicates = []; - this.clientMissing = []; - this.serverMissing = []; - this.serverDeleted = []; - this.serverUnexpected = []; - this.differences = []; - } - - /** - * Produce a list summarizing problems found. Each entry contains {name, count}, - * where name is the field name for the problem, and count is the number of times - * the problem was encountered. - * - * Validation has failed if all counts are not 0. - */ - getSummary() { - return [ - { name: "clientMissing", count: this.clientMissing.length }, - { name: "serverMissing", count: this.serverMissing.length }, - { name: "serverDeleted", count: this.serverDeleted.length }, - { name: "serverUnexpected", count: this.serverUnexpected.length }, - { name: "differences", count: this.differences.length }, - { name: "missingIDs", count: this.missingIDs }, - { name: "duplicates", count: this.duplicates.length } - ]; - } -} - -class CollectionValidator { - // Construct a generic collection validator. This is intended to be called by - // subclasses. - // - name: Name of the engine - // - idProp: Property that identifies a record. That is, if a client and server - // record have the same value for the idProp property, they should be - // compared against eachother. - // - props: Array of properties that should be compared - constructor(name, idProp, props) { - this.name = name; - this.props = props; - this.idProp = idProp; - } - - // Should a custom ProblemData type be needed, return it here. - emptyProblemData() { - return new CollectionProblemData(); - } - - getServerItems(engine) { - let collection = engine.itemSource(); - let collectionKey = engine.service.collectionKeys.keyForCollection(engine.name); - collection.full = true; - let items = []; - collection.recordHandler = function(item) { - item.decrypt(collectionKey); - items.push(item.cleartext); - }; - let resp = collection.getBatched(); - if (!resp.success) { - throw resp; - } - return items; - } - - // Should return a promise that resolves to an array of client items. - getClientItems() { - return Promise.reject("Must implement"); - } - - // Turn the client item into something that can be compared with the server item, - // and is also safe to mutate. - normalizeClientItem(item) { - return Cu.cloneInto(item, {}); - } - - // Turn the server item into something that can be easily compared with the client - // items. - normalizeServerItem(item) { - return item; - } - - // Return whether or not a server item should be present on the client. Expected - // to be overridden. - clientUnderstands(item) { - return true; - } - - // Return whether or not a client item should be present on the server. Expected - // to be overridden - syncedByClient(item) { - return true; - } - - // Compare the server item and the client item, and return a list of property - // names that are different. Can be overridden if needed. - getDifferences(client, server) { - let differences = []; - for (let prop of this.props) { - let clientProp = client[prop]; - let serverProp = server[prop]; - if ((clientProp || "") !== (serverProp || "")) { - differences.push(prop); - } - } - return differences; - } - - // Returns an object containing - // problemData: an instance of the class returned by emptyProblemData(), - // clientRecords: Normalized client records - // records: Normalized server records, - // deletedRecords: Array of ids that were marked as deleted by the server. - compareClientWithServer(clientItems, serverItems) { - clientItems = clientItems.map(item => this.normalizeClientItem(item)); - serverItems = serverItems.map(item => this.normalizeServerItem(item)); - let problems = this.emptyProblemData(); - let seenServer = new Map(); - let serverDeleted = new Set(); - let allRecords = new Map(); - - for (let record of serverItems) { - let id = record[this.idProp]; - if (!id) { - ++problems.missingIDs; - continue; - } - if (record.deleted) { - serverDeleted.add(record); - } else { - let possibleDupe = seenServer.get(id); - if (possibleDupe) { - problems.duplicates.push(id); - } else { - seenServer.set(id, record); - allRecords.set(id, { server: record, client: null, }); - } - record.understood = this.clientUnderstands(record); - } - } - - let recordPairs = []; - let seenClient = new Map(); - for (let record of clientItems) { - let id = record[this.idProp]; - record.shouldSync = this.syncedByClient(record); - seenClient.set(id, record); - let combined = allRecords.get(id); - if (combined) { - combined.client = record; - } else { - allRecords.set(id, { client: record, server: null }); - } - } - - for (let [id, { server, client }] of allRecords) { - if (!client && !server) { - throw new Error("Impossible: no client or server record for " + id); - } else if (server && !client) { - if (server.understood) { - problems.clientMissing.push(id); - } - } else if (client && !server) { - if (client.shouldSync) { - problems.serverMissing.push(id); - } - } else { - if (!client.shouldSync) { - if (!problems.serverUnexpected.includes(id)) { - problems.serverUnexpected.push(id); - } - continue; - } - let differences = this.getDifferences(client, server); - if (differences && differences.length) { - problems.differences.push({ id, differences }); - } - } - } - return { - problemData: problems, - clientRecords: clientItems, - records: serverItems, - deletedRecords: [...serverDeleted] - }; - } -} - -// Default to 0, some engines may override. -CollectionValidator.prototype.version = 0; diff --git a/services/sync/modules/constants.js b/services/sync/modules/constants.js index f70bbd61c..c8d66d921 100644 --- a/services/sync/modules/constants.js +++ b/services/sync/modules/constants.js @@ -4,8 +4,7 @@ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ // Process each item in the "constants hash" to add to "global" and give a name -this.EXPORTED_SYMBOLS = []; -for (let [key, val] of Object.entries({ +this.EXPORTED_SYMBOLS = [((this[key] = val), key) for ([key, val] in Iterator({ WEAVE_VERSION: "@weave_version@", @@ -45,7 +44,7 @@ MAX_IGNORE_ERROR_COUNT: 5, // Backoff intervals MINIMUM_BACKOFF_INTERVAL: 15 * 60 * 1000, // 15 minutes -MAXIMUM_BACKOFF_INTERVAL: 8 * 60 * 60 * 1000, // 8 hours +MAXIMUM_BACKOFF_INTERVAL: 8 * 60 * 60 * 1000, // 8 hours // HMAC event handling timeout. // 10 minutes: a compromise between the multi-desktop sync interval @@ -76,10 +75,6 @@ PASSWORDS_STORE_BATCH_SIZE: 50, // same as MOBILE_BATCH_SIZE ADDONS_STORE_BATCH_SIZE: 1000000, // process all addons at once APPS_STORE_BATCH_SIZE: 50, // same as MOBILE_BATCH_SIZE -// Default batch size for download batching -// (how many records are fetched at a time from the server when batching is used). -DEFAULT_DOWNLOAD_BATCH_SIZE: 1000, - // score thresholds for early syncs SINGLE_USER_THRESHOLD: 1000, MULTI_DEVICE_THRESHOLD: 300, @@ -98,16 +93,13 @@ SCORE_UPDATE_DELAY: 100, // observed spurious idle/back events and short enough to pre-empt user activity. IDLE_OBSERVER_BACK_DELAY: 100, -// Max number of records or bytes to upload in a single POST - we'll do multiple POSTS if either -// MAX_UPLOAD_RECORDS or MAX_UPLOAD_BYTES is hit) +// Number of records to upload in a single POST (multiple POSTS if exceeded) +// FIXME: Record size limit is 256k (new cluster), so this can be quite large! +// (Bug 569295) MAX_UPLOAD_RECORDS: 100, -MAX_UPLOAD_BYTES: 1024 * 1023, // just under 1MB MAX_HISTORY_UPLOAD: 5000, MAX_HISTORY_DOWNLOAD: 5000, -// TTL of the message sent to another device when sending a tab -NOTIFY_TAB_SENT_TTL_SECS: 1 * 3600, // 1 hour - // Top-level statuses: STATUS_OK: "success.status_ok", SYNC_FAILED: "error.sync.failed", @@ -130,6 +122,7 @@ LOGIN_FAILED_NETWORK_ERROR: "error.login.reason.network", LOGIN_FAILED_SERVER_ERROR: "error.login.reason.server", LOGIN_FAILED_INVALID_PASSPHRASE: "error.login.reason.recoverykey", LOGIN_FAILED_LOGIN_REJECTED: "error.login.reason.account", +LOGIN_FAILED_NOT_READY: "error.login.reason.initializing", // sync failure status codes METARECORD_DOWNLOAD_FAIL: "error.sync.reason.metarecord_download_fail", @@ -152,8 +145,6 @@ ENGINE_UNKNOWN_FAIL: "error.engine.reason.unknown_fail", ENGINE_APPLY_FAIL: "error.engine.reason.apply_fail", ENGINE_METARECORD_DOWNLOAD_FAIL: "error.engine.reason.metarecord_download_fail", ENGINE_METARECORD_UPLOAD_FAIL: "error.engine.reason.metarecord_upload_fail", -// an upload failure where the batch was interrupted with a 412 -ENGINE_BATCH_INTERRUPTED: "error.engine.reason.batch_interrupted", JPAKE_ERROR_CHANNEL: "jpake.error.channel", JPAKE_ERROR_NETWORK: "jpake.error.network", @@ -181,7 +172,7 @@ kSyncBackoffNotMet: "Trying to sync before the server said it kFirstSyncChoiceNotMade: "User has not selected an action for first sync", // Application IDs -FIREFOX_ID: "{ec8030f7-c20a-464f-9b0e-13a3a9e97384}", +FIREFOX_ID: "{8de7fcbb-c55c-4fbe-bfc5-fc555c87dbc4}", FENNEC_ID: "{a23983c0-fd0e-11dc-95ff-0800200c9a66}", SEAMONKEY_ID: "{92650c4d-4b8e-4d2a-b7eb-24ecf4f6b63a}", TEST_HARNESS_ID: "xuth@mozilla.org", @@ -189,10 +180,6 @@ TEST_HARNESS_ID: "xuth@mozilla.org", MIN_PP_LENGTH: 12, MIN_PASS_LENGTH: 8, -DEVICE_TYPE_DESKTOP: "desktop", -DEVICE_TYPE_MOBILE: "mobile", +LOG_DATE_FORMAT: "%Y-%m-%d %H:%M:%S", -})) { - this[key] = val; - this.EXPORTED_SYMBOLS.push(key); -} +}))]; diff --git a/services/sync/modules/engines.js b/services/sync/modules/engines.js index 1eaa1863a..eabddae2b 100644 --- a/services/sync/modules/engines.js +++ b/services/sync/modules/engines.js @@ -7,24 +7,21 @@ this.EXPORTED_SYMBOLS = [ "Engine", "SyncEngine", "Tracker", - "Store", - "Changeset" + "Store" ]; -var {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components; +const {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components; Cu.import("resource://services-common/async.js"); Cu.import("resource://gre/modules/Log.jsm"); Cu.import("resource://services-common/observers.js"); +Cu.import("resource://services-common/utils.js"); Cu.import("resource://services-sync/constants.js"); Cu.import("resource://services-sync/identity.js"); Cu.import("resource://services-sync/record.js"); Cu.import("resource://services-sync/resource.js"); Cu.import("resource://services-sync/util.js"); -XPCOMUtils.defineLazyModuleGetter(this, "fxAccounts", - "resource://gre/modules/FxAccounts.jsm"); - /* * Trackers are associated with a single engine and deal with * listening for changes to their particular data type. @@ -107,7 +104,7 @@ Tracker.prototype = { Utils.jsonLoad("changes/" + this.file, this, function(json) { if (json && (typeof(json) == "object")) { this.changedIDs = json; - } else if (json !== null) { + } else { this._log.warn("Changed IDs file " + this.file + " contains non-object value."); json = null; } @@ -132,30 +129,26 @@ Tracker.prototype = { this._ignored.splice(index, 1); }, - _saveChangedID(id, when) { - this._log.trace(`Adding changed ID: ${id}, ${JSON.stringify(when)}`); - this.changedIDs[id] = when; - this.saveChangedIDs(this.onSavedChangedIDs); - }, - addChangedID: function (id, when) { if (!id) { this._log.warn("Attempted to add undefined ID to tracker"); return false; } - if (this.ignoreAll || this._ignored.includes(id)) { + if (this.ignoreAll || (id in this._ignored)) { return false; } // Default to the current time in seconds if no time is provided. if (when == null) { - when = this._now(); + when = Math.floor(Date.now() / 1000); } // Add/update the entry if we have a newer time. if ((this.changedIDs[id] || -Infinity) < when) { - this._saveChangedID(id, when); + this._log.trace("Adding changed ID: " + id + ", " + when); + this.changedIDs[id] = when; + this.saveChangedIDs(this.onSavedChangedIDs); } return true; @@ -166,9 +159,8 @@ Tracker.prototype = { this._log.warn("Attempted to remove undefined ID to tracker"); return false; } - if (this.ignoreAll || this._ignored.includes(id)) { + if (this.ignoreAll || (id in this._ignored)) return false; - } if (this.changedIDs[id] != null) { this._log.trace("Removing changed ID " + id); delete this.changedIDs[id]; @@ -183,10 +175,6 @@ Tracker.prototype = { this.saveChangedIDs(); }, - _now() { - return Date.now() / 1000; - }, - _isTracking: false, // Override these in your subclasses. @@ -311,21 +299,17 @@ Store.prototype = { */ applyIncomingBatch: function (records) { let failed = []; - for (let record of records) { + for each (let record in records) { try { this.applyIncoming(record); + } catch (ex if (ex.code == Engine.prototype.eEngineAbortApplyIncoming)) { + // This kind of exception should have a 'cause' attribute, which is an + // originating exception. + // ex.cause will carry its stack with it when rethrown. + throw ex.cause; } catch (ex) { - if (ex.code == Engine.prototype.eEngineAbortApplyIncoming) { - // This kind of exception should have a 'cause' attribute, which is an - // originating exception. - // ex.cause will carry its stack with it when rethrown. - throw ex.cause; - } - if (Async.isShutdownException(ex)) { - throw ex; - } - this._log.warn("Failed to apply incoming record " + record.id, ex); - this.engine._noteApplyFailure(); + this._log.warn("Failed to apply incoming record " + record.id); + this._log.warn("Encountered exception: " + Utils.exceptionStr(ex)); failed.push(record.id); } }; @@ -499,11 +483,7 @@ EngineManager.prototype = { }, getAll: function () { - let engines = []; - for (let [, engine] of Object.entries(this._engines)) { - engines.push(engine); - } - return engines; + return [engine for ([name, engine] in Iterator(this._engines))]; }, /** @@ -516,7 +496,7 @@ EngineManager.prototype = { }, get enabledEngineNames() { - return this.getEnabled().map(e => e.name); + return [e.name for each (e in this.getEnabled())]; }, persistDeclined: function () { @@ -593,11 +573,16 @@ EngineManager.prototype = { this._engines[name] = engine; } } catch (ex) { + this._log.error(CommonUtils.exceptionStr(ex)); + + let mesg = ex.message ? ex.message : ex; let name = engineObject || ""; name = name.prototype || ""; name = name.name || ""; - this._log.error(`Could not initialize engine ${name}`, ex); + let out = "Could not initialize engine '" + name + "': " + mesg; + this._log.error(out); + return engineObject; } }, @@ -643,17 +628,16 @@ Engine.prototype = { // Signal to the engine that processing further records is pointless. eEngineAbortApplyIncoming: "error.engine.abort.applyincoming", - // Should we keep syncing if we find a record that cannot be uploaded (ever)? - // If this is false, we'll throw, otherwise, we'll ignore the record and - // continue. This currently can only happen due to the record being larger - // than the record upload limit. - allowSkippedRecord: true, - get prefName() { return this.name; }, get enabled() { + // XXX: Disable non-functional add-ons syncing for the time being + // This check can go away when add-on syncing is addressed + if (this.prefName == "addons") + return false; + return Svc.Prefs.get("engine." + this.prefName, false); }, @@ -711,15 +695,6 @@ Engine.prototype = { wipeClient: function () { this._notify("wipe-client", this.name, this._wipeClient)(); - }, - - /** - * If one exists, initialize and return a validator for this engine (which - * must have a `validate(engine)` method that returns a promise to an object - * with a getSummary method). Otherwise return null. - */ - getValidator: function () { - return null; } }; @@ -813,11 +788,7 @@ SyncEngine.prototype = { return this._toFetch; }, set toFetch(val) { - let cb = (error) => { - if (error) { - this._log.error("Failed to read JSON records to fetch", error); - } - } + let cb = (error) => this._log.error(Utils.exceptionStr(error)); // Coerce the array to a string for more efficient comparison. if (val + "" == this._toFetch) { return; @@ -842,13 +813,7 @@ SyncEngine.prototype = { return this._previousFailed; }, set previousFailed(val) { - let cb = (error) => { - if (error) { - this._log.error("Failed to set previousFailed", error); - } else { - this._log.debug("Successfully wrote previousFailed."); - } - } + let cb = (error) => this._log.error(Utils.exceptionStr(error)); // Coerce the array to a string for more efficient comparison. if (val + "" == this._previousFailed) { return; @@ -881,8 +846,9 @@ SyncEngine.prototype = { }, /* - * Returns a changeset for this sync. Engine implementations can override this - * method to bypass the tracker for certain or all changed items. + * Returns a mapping of IDs -> changed timestamp. Engine implementations + * can override this method to bypass the tracker for certain or all + * changed items. */ getChangedIDs: function () { return this._tracker.changedIDs; @@ -950,16 +916,20 @@ SyncEngine.prototype = { // this._modified to the tracker. this.lastSyncLocal = Date.now(); if (this.lastSync) { - this._modified = this.pullNewChanges(); + this._modified = this.getChangedIDs(); } else { + // Mark all items to be uploaded, but treat them as changed from long ago this._log.debug("First sync, uploading all items"); - this._modified = this.pullAllChanges(); + this._modified = {}; + for (let id in this._store.getAllIDs()) { + this._modified[id] = 0; + } } // Clear the tracker now. If the sync fails we'll add the ones we failed // to upload back. this._tracker.clearChangedIDs(); - this._log.info(this._modified.count() + + this._log.info(Object.keys(this._modified).length + " outgoing items pre-reconciliation"); // Keep track of what to delete at the end of sync @@ -970,7 +940,7 @@ SyncEngine.prototype = { * A tiny abstraction to make it easier to test incoming record * application. */ - itemSource: function () { + _itemSource: function () { return new Collection(this.engineURL, this._recordObj, this.service); }, @@ -987,7 +957,7 @@ SyncEngine.prototype = { let isMobile = (Svc.Prefs.get("client.type") == "mobile"); if (!newitems) { - newitems = this.itemSource(); + newitems = this._itemSource(); } if (this._defaultSort) { @@ -1024,12 +994,10 @@ SyncEngine.prototype = { try { failed = failed.concat(this._store.applyIncomingBatch(applyBatch)); } catch (ex) { - if (Async.isShutdownException(ex)) { - throw ex; - } // Catch any error that escapes from applyIncomingBatch. At present // those will all be abort events. - this._log.warn("Got exception, aborting processIncoming", ex); + this._log.warn("Got exception " + Utils.exceptionStr(ex) + + ", aborting processIncoming."); aborting = ex; } this._tracker.ignoreAll = false; @@ -1074,10 +1042,7 @@ SyncEngine.prototype = { try { try { item.decrypt(key); - } catch (ex) { - if (!Utils.isHMACMismatch(ex)) { - throw ex; - } + } catch (ex if Utils.isHMACMismatch(ex)) { let strategy = self.handleHMACMismatch(item, true); if (strategy == SyncEngine.kRecoveryStrategy.retry) { // You only get one retry. @@ -1087,10 +1052,7 @@ SyncEngine.prototype = { key = self.service.collectionKeys.keyForCollection(self.name); item.decrypt(key); strategy = null; - } catch (ex) { - if (!Utils.isHMACMismatch(ex)) { - throw ex; - } + } catch (ex if Utils.isHMACMismatch(ex)) { strategy = self.handleHMACMismatch(item, false); } } @@ -1103,8 +1065,7 @@ SyncEngine.prototype = { self._log.debug("Ignoring second retry suggestion."); // Fall through to error case. case SyncEngine.kRecoveryStrategy.error: - self._log.warn("Error decrypting record", ex); - self._noteApplyFailure(); + self._log.warn("Error decrypting record: " + Utils.exceptionStr(ex)); failed.push(item.id); return; case SyncEngine.kRecoveryStrategy.ignore: @@ -1114,11 +1075,7 @@ SyncEngine.prototype = { } } } catch (ex) { - if (Async.isShutdownException(ex)) { - throw ex; - } - self._log.warn("Error decrypting record", ex); - self._noteApplyFailure(); + self._log.warn("Error decrypting record: " + Utils.exceptionStr(ex)); failed.push(item.id); return; } @@ -1126,20 +1083,15 @@ SyncEngine.prototype = { let shouldApply; try { shouldApply = self._reconcile(item); + } catch (ex if (ex.code == Engine.prototype.eEngineAbortApplyIncoming)) { + self._log.warn("Reconciliation failed: aborting incoming processing."); + failed.push(item.id); + aborting = ex.cause; } catch (ex) { - if (ex.code == Engine.prototype.eEngineAbortApplyIncoming) { - self._log.warn("Reconciliation failed: aborting incoming processing."); - self._noteApplyFailure(); - failed.push(item.id); - aborting = ex.cause; - } else if (!Async.isShutdownException(ex)) { - self._log.warn("Failed to reconcile incoming record " + item.id, ex); - self._noteApplyFailure(); - failed.push(item.id); - return; - } else { - throw ex; - } + self._log.warn("Failed to reconcile incoming record " + item.id); + self._log.warn("Encountered exception: " + Utils.exceptionStr(ex)); + failed.push(item.id); + return; } if (shouldApply) { @@ -1158,7 +1110,7 @@ SyncEngine.prototype = { // Only bother getting data from the server if there's new things if (this.lastModified == null || this.lastModified > this.lastSync) { - let resp = newitems.getBatched(); + let resp = newitems.get(); doApplyBatchAndPersistFailed.call(this); if (!resp.success) { resp.failureCode = ENGINE_DOWNLOAD_FAIL; @@ -1243,13 +1195,7 @@ SyncEngine.prototype = { // Apply remaining items. doApplyBatchAndPersistFailed.call(this); - count.newFailed = this.previousFailed.reduce((count, engine) => { - if (failedInPreviousSync.indexOf(engine) == -1) { - count++; - this._noteApplyNewFailure(); - } - return count; - }, 0); + count.newFailed = Utils.arraySub(this.previousFailed, failedInPreviousSync).length; count.succeeded = Math.max(0, count.applied - count.failed); this._log.info(["Records:", count.applied, "applied,", @@ -1260,14 +1206,6 @@ SyncEngine.prototype = { Observers.notify("weave:engine:sync:applied", count, this.name); }, - _noteApplyFailure: function () { - // here would be a good place to record telemetry... - }, - - _noteApplyNewFailure: function () { - // here would be a good place to record telemetry... - }, - /** * Find a GUID of an item that is a duplicate of the incoming item but happens * to have a different GUID @@ -1278,16 +1216,6 @@ SyncEngine.prototype = { // By default, assume there's no dupe items for the engine }, - // Called when the server has a record marked as deleted, but locally we've - // changed it more recently than the deletion. If we return false, the - // record will be deleted locally. If we return true, we'll reupload the - // record to the server -- any extra work that's needed as part of this - // process should be done at this point (such as mark the record's parent - // for reuploading in the case of bookmarks). - _shouldReviveRemotelyDeletedRecord(remoteItem) { - return true; - }, - _deleteId: function (id) { this._tracker.removeChangedID(id); @@ -1298,18 +1226,6 @@ SyncEngine.prototype = { this._delete.ids.push(id); }, - _switchItemToDupe(localDupeGUID, incomingItem) { - // The local, duplicate ID is always deleted on the server. - this._deleteId(localDupeGUID); - - // We unconditionally change the item's ID in case the engine knows of - // an item but doesn't expose it through itemExists. If the API - // contract were stronger, this could be changed. - this._log.debug("Switching local ID to incoming: " + localDupeGUID + " -> " + - incomingItem.id); - this._store.changeItemID(localDupeGUID, incomingItem.id); - }, - /** * Reconcile incoming record with local state. * @@ -1329,12 +1245,12 @@ SyncEngine.prototype = { // because some state may change during the course of this function and we // need to operate on the original values. let existsLocally = this._store.itemExists(item.id); - let locallyModified = this._modified.has(item.id); + let locallyModified = item.id in this._modified; // TODO Handle clock drift better. Tracked in bug 721181. let remoteAge = AsyncResource.serverTime - item.modified; let localAge = locallyModified ? - (Date.now() / 1000 - this._modified.getModifiedTimestamp(item.id)) : null; + (Date.now() / 1000 - this._modified[item.id]) : null; let remoteIsNewer = remoteAge < localAge; this._log.trace("Reconciling " + item.id + ". exists=" + @@ -1363,18 +1279,15 @@ SyncEngine.prototype = { "exists and isn't modified."); return true; } - this._log.trace("Incoming record is deleted but we had local changes."); - if (remoteIsNewer) { - this._log.trace("Remote record is newer -- deleting local record."); - return true; - } - // If the local record is newer, we defer to individual engines for - // how to handle this. By default, we revive the record. - let willRevive = this._shouldReviveRemotelyDeletedRecord(item); - this._log.trace("Local record is newer -- reviving? " + willRevive); - - return !willRevive; + // TODO As part of bug 720592, determine whether we should do more here. + // In the case where the local changes are newer, it is quite possible + // that the local client will restore data a remote client had tried to + // delete. There might be a good reason for that delete and it might be + // enexpected for this client to restore that data. + this._log.trace("Incoming record is deleted but we had local changes. " + + "Applying the youngest record."); + return remoteIsNewer; } // At this point the incoming record is not for a deletion and must have @@ -1386,32 +1299,40 @@ SyncEngine.prototype = { // refresh the metadata collected above. See bug 710448 for the history // of this logic. if (!existsLocally) { - let localDupeGUID = this._findDupe(item); - if (localDupeGUID) { - this._log.trace("Local item " + localDupeGUID + " is a duplicate for " + + let dupeID = this._findDupe(item); + if (dupeID) { + this._log.trace("Local item " + dupeID + " is a duplicate for " + "incoming item " + item.id); + // The local, duplicate ID is always deleted on the server. + this._deleteId(dupeID); + // The current API contract does not mandate that the ID returned by // _findDupe() actually exists. Therefore, we have to perform this // check. - existsLocally = this._store.itemExists(localDupeGUID); + existsLocally = this._store.itemExists(dupeID); + + // We unconditionally change the item's ID in case the engine knows of + // an item but doesn't expose it through itemExists. If the API + // contract were stronger, this could be changed. + this._log.debug("Switching local ID to incoming: " + dupeID + " -> " + + item.id); + this._store.changeItemID(dupeID, item.id); // If the local item was modified, we carry its metadata forward so // appropriate reconciling can be performed. - if (this._modified.has(localDupeGUID)) { + if (dupeID in this._modified) { locallyModified = true; - localAge = this._tracker._now() - this._modified.getModifiedTimestamp(localDupeGUID); + localAge = Date.now() / 1000 - this._modified[dupeID]; remoteIsNewer = remoteAge < localAge; - this._modified.swap(localDupeGUID, item.id); + this._modified[item.id] = this._modified[dupeID]; + delete this._modified[dupeID]; } else { locallyModified = false; localAge = null; } - // Tell the engine to do whatever it needs to switch the items. - this._switchItemToDupe(localDupeGUID, item); - this._log.debug("Local item after duplication: age=" + localAge + "; modified=" + locallyModified + "; exists=" + existsLocally); @@ -1440,7 +1361,7 @@ SyncEngine.prototype = { if (remoteIsNewer) { this._log.trace("Applying incoming because local item was deleted " + "before the incoming item was changed."); - this._modified.delete(item.id); + delete this._modified[item.id]; return true; } @@ -1466,7 +1387,7 @@ SyncEngine.prototype = { this._log.trace("Ignoring incoming item because the local item is " + "identical."); - this._modified.delete(item.id); + delete this._modified[item.id]; return false; } @@ -1491,97 +1412,69 @@ SyncEngine.prototype = { _uploadOutgoing: function () { this._log.trace("Uploading local changes to server."); - let modifiedIDs = this._modified.ids(); + let modifiedIDs = Object.keys(this._modified); if (modifiedIDs.length) { this._log.trace("Preparing " + modifiedIDs.length + " outgoing records"); - let counts = { sent: modifiedIDs.length, failed: 0 }; - // collection we'll upload let up = new Collection(this.engineURL, null, this.service); + let count = 0; - let failed = []; - let successful = []; - let handleResponse = (resp, batchOngoing = false) => { - // Note: We don't want to update this.lastSync, or this._modified until - // the batch is complete, however we want to remember success/failure - // indicators for when that happens. + // Upload what we've got so far in the collection + let doUpload = Utils.bind2(this, function(desc) { + this._log.info("Uploading " + desc + " of " + modifiedIDs.length + + " records"); + let resp = up.post(); if (!resp.success) { this._log.debug("Uploading records failed: " + resp); - resp.failureCode = resp.status == 412 ? ENGINE_BATCH_INTERRUPTED : ENGINE_UPLOAD_FAIL; + resp.failureCode = ENGINE_UPLOAD_FAIL; throw resp; } // Update server timestamp from the upload. - failed = failed.concat(Object.keys(resp.obj.failed)); - successful = successful.concat(resp.obj.success); - - if (batchOngoing) { - // Nothing to do yet - return; - } - // Advance lastSync since we've finished the batch. let modified = resp.headers["x-weave-timestamp"]; - if (modified > this.lastSync) { + if (modified > this.lastSync) this.lastSync = modified; - } - if (failed.length && this._log.level <= Log.Level.Debug) { + + let failed_ids = Object.keys(resp.obj.failed); + if (failed_ids.length) this._log.debug("Records that will be uploaded again because " + "the server couldn't store them: " - + failed.join(", ")); - } - - counts.failed += failed.length; + + failed_ids.join(", ")); - for (let id of successful) { - this._modified.delete(id); + // Clear successfully uploaded objects. + for each (let id in resp.obj.success) { + delete this._modified[id]; } - this._onRecordsWritten(successful, failed); - - // clear for next batch - failed.length = 0; - successful.length = 0; - }; - - let postQueue = up.newPostQueue(this._log, this.lastSync, handleResponse); + up.clearRecords(); + }); - for (let id of modifiedIDs) { - let out; - let ok = false; + for each (let id in modifiedIDs) { try { - out = this._createRecord(id); + let out = this._createRecord(id); if (this._log.level <= Log.Level.Trace) this._log.trace("Outgoing: " + out); out.encrypt(this.service.collectionKeys.keyForCollection(this.name)); - ok = true; - } catch (ex) { - if (Async.isShutdownException(ex)) { - throw ex; - } - this._log.warn("Error creating record", ex); + up.pushData(out); } - if (ok) { - let { enqueued, error } = postQueue.enqueue(out); - if (!enqueued) { - ++counts.failed; - if (!this.allowSkippedRecord) { - throw error; - } - } + catch(ex) { + this._log.warn("Error creating record: " + Utils.exceptionStr(ex)); } + + // Partial upload + if ((++count % MAX_UPLOAD_RECORDS) == 0) + doUpload((count - MAX_UPLOAD_RECORDS) + " - " + count + " out"); + this._store._sleep(0); } - postQueue.flush(true); - Observers.notify("weave:engine:sync:uploaded", counts, this.name); - } - }, - _onRecordsWritten(succeeded, failed) { - // Implement this method to take specific actions against successfully - // uploaded records and failed records. + // Final upload + if (count % MAX_UPLOAD_RECORDS > 0) + doUpload(count >= MAX_UPLOAD_RECORDS ? "last batch" : "all"); + } }, // Any cleanup necessary. @@ -1596,7 +1489,7 @@ SyncEngine.prototype = { coll.delete(); }); - for (let [key, val] of Object.entries(this._delete)) { + for (let [key, val] in Iterator(this._delete)) { // Remove the key for future uses delete this._delete[key]; @@ -1619,8 +1512,10 @@ SyncEngine.prototype = { } // Mark failed WBOs as changed again so they are reuploaded next time. - this.trackRemainingChanges(); - this._modified.clear(); + for (let [id, when] in Iterator(this._modified)) { + this._tracker.addChangedID(id, when); + } + this._modified = {}; }, _sync: function () { @@ -1656,11 +1551,9 @@ SyncEngine.prototype = { try { this._log.trace("Trying to decrypt a record from the server.."); test.get(); - } catch (ex) { - if (Async.isShutdownException(ex)) { - throw ex; - } - this._log.debug("Failed test decrypt", ex); + } + catch(ex) { + this._log.debug("Failed test decrypt: " + Utils.exceptionStr(ex)); } return canDecrypt; @@ -1706,108 +1599,5 @@ SyncEngine.prototype = { return (this.service.handleHMACEvent() && mayRetry) ? SyncEngine.kRecoveryStrategy.retry : SyncEngine.kRecoveryStrategy.error; - }, - - /** - * Returns a changeset containing all items in the store. The default - * implementation returns a changeset with timestamps from long ago, to - * ensure we always use the remote version if one exists. - * - * This function is only called for the first sync. Subsequent syncs call - * `pullNewChanges`. - * - * @return A `Changeset` object. - */ - pullAllChanges() { - let changeset = new Changeset(); - for (let id in this._store.getAllIDs()) { - changeset.set(id, 0); - } - return changeset; - }, - - /* - * Returns a changeset containing entries for all currently tracked items. - * The default implementation returns a changeset with timestamps indicating - * when the item was added to the tracker. - * - * @return A `Changeset` object. - */ - pullNewChanges() { - return new Changeset(this.getChangedIDs()); - }, - - /** - * Adds all remaining changeset entries back to the tracker, typically for - * items that failed to upload. This method is called at the end of each sync. - * - */ - trackRemainingChanges() { - for (let [id, change] of this._modified.entries()) { - this._tracker.addChangedID(id, change); - } - }, -}; - -/** - * A changeset is created for each sync in `Engine::get{Changed, All}IDs`, - * and stores opaque change data for tracked IDs. The default implementation - * only records timestamps, though engines can extend this to store additional - * data for each entry. - */ -class Changeset { - // Creates a changeset with an initial set of tracked entries. - constructor(changes = {}) { - this.changes = changes; - } - - // Returns the last modified time, in seconds, for an entry in the changeset. - // `id` is guaranteed to be in the set. - getModifiedTimestamp(id) { - return this.changes[id]; - } - - // Adds a change for a tracked ID to the changeset. - set(id, change) { - this.changes[id] = change; - } - - // Indicates whether an entry is in the changeset. - has(id) { - return id in this.changes; } - - // Deletes an entry from the changeset. Used to clean up entries for - // reconciled and successfully uploaded records. - delete(id) { - delete this.changes[id]; - } - - // Swaps two entries in the changeset. Used when reconciling duplicates that - // have local changes. - swap(oldID, newID) { - this.changes[newID] = this.changes[oldID]; - delete this.changes[oldID]; - } - - // Returns an array of all tracked IDs in this changeset. - ids() { - return Object.keys(this.changes); - } - - // Returns an array of `[id, change]` tuples. Used to repopulate the tracker - // with entries for failed uploads at the end of a sync. - entries() { - return Object.entries(this.changes); - } - - // Returns the number of entries in this changeset. - count() { - return this.ids().length; - } - - // Clears the changeset. - clear() { - this.changes = {}; - } -} +}; diff --git a/services/sync/modules/engines/addons.js b/services/sync/modules/engines/addons.js index 01dab58d1..ab3131c30 100644 --- a/services/sync/modules/engines/addons.js +++ b/services/sync/modules/engines/addons.js @@ -25,18 +25,15 @@ * * Synchronization is influenced by the following preferences: * + * - services.sync.addons.ignoreRepositoryChecking * - services.sync.addons.ignoreUserEnabledChanges * - services.sync.addons.trustedSourceHostnames * - * and also influenced by whether addons have repository caching enabled and - * whether they allow installation of addons from insecure options (both of - * which are themselves influenced by the "extensions." pref branch) - * * See the documentation in services-sync.js for the behavior of these prefs. */ "use strict"; -var {classes: Cc, interfaces: Ci, utils: Cu} = Components; +const {classes: Cc, interfaces: Ci, utils: Cu} = Components; Cu.import("resource://services-sync/addonutils.js"); Cu.import("resource://services-sync/addonsreconciler.js"); @@ -44,7 +41,6 @@ Cu.import("resource://services-sync/engines.js"); Cu.import("resource://services-sync/record.js"); Cu.import("resource://services-sync/util.js"); Cu.import("resource://services-sync/constants.js"); -Cu.import("resource://services-sync/collection_validator.js"); Cu.import("resource://services-common/async.js"); Cu.import("resource://gre/modules/Preferences.jsm"); @@ -54,7 +50,7 @@ XPCOMUtils.defineLazyModuleGetter(this, "AddonManager", XPCOMUtils.defineLazyModuleGetter(this, "AddonRepository", "resource://gre/modules/addons/AddonRepository.jsm"); -this.EXPORTED_SYMBOLS = ["AddonsEngine", "AddonValidator"]; +this.EXPORTED_SYMBOLS = ["AddonsEngine"]; // 7 days in milliseconds. const PRUNE_ADDON_CHANGES_THRESHOLD = 60 * 60 * 24 * 7 * 1000; @@ -154,7 +150,7 @@ AddonsEngine.prototype = { */ getChangedIDs: function getChangedIDs() { let changes = {}; - for (let [id, modified] of Object.entries(this._tracker.changedIDs)) { + for (let [id, modified] in Iterator(this._tracker.changedIDs)) { changes[id] = modified; } @@ -164,7 +160,7 @@ AddonsEngine.prototype = { // we assume this function is only called from within a sync. let reconcilerChanges = this._reconciler.getChangesSinceDate(lastSyncDate); let addons = this._reconciler.addons; - for (let change of reconcilerChanges) { + for each (let change in reconcilerChanges) { let changeTime = change[0]; let id = change[2]; @@ -177,7 +173,7 @@ AddonsEngine.prototype = { continue; } - if (!this.isAddonSyncable(addons[id])) { + if (!this._store.isAddonSyncable(addons[id])) { continue; } @@ -235,10 +231,6 @@ AddonsEngine.prototype = { let cb = Async.makeSpinningCallback(); this._reconciler.refreshGlobalState(cb); cb.wait(); - }, - - isAddonSyncable(addon, ignoreRepoCheck) { - return this._store.isAddonSyncable(addon, ignoreRepoCheck); } }; @@ -286,14 +278,6 @@ AddonsStore.prototype = { } } - // Ignore incoming records for which an existing non-syncable addon - // exists. - let existingMeta = this.reconciler.addons[record.addonID]; - if (existingMeta && !this.isAddonSyncable(existingMeta)) { - this._log.info("Ignoring incoming record for an existing but non-syncable addon", record.addonID); - return; - } - Store.prototype.applyIncoming.call(this, record); }, @@ -307,23 +291,15 @@ AddonsStore.prototype = { id: record.addonID, syncGUID: record.id, enabled: record.enabled, - requireSecureURI: this._extensionsPrefs.get("install.requireSecureOrigin", true), + requireSecureURI: !Svc.Prefs.get("addons.ignoreRepositoryChecking", false), }], cb); // This will throw if there was an error. This will get caught by the sync // engine and the record will try to be applied later. let results = cb.wait(); - if (results.skipped.includes(record.addonID)) { - this._log.info("Add-on skipped: " + record.addonID); - // Just early-return for skipped addons - we don't want to arrange to - // try again next time because the condition that caused up to skip - // will remain true for this addon forever. - return; - } - let addon; - for (let a of results.addons) { + for each (let a in results.addons) { if (a.id == record.addonID) { addon = a; break; @@ -467,8 +443,7 @@ AddonsStore.prototype = { let ids = {}; let addons = this.reconciler.addons; - for (let id in addons) { - let addon = addons[id]; + for each (let addon in addons) { if (this.isAddonSyncable(addon)) { ids[addon.guid] = true; } @@ -499,7 +474,7 @@ AddonsStore.prototype = { } this._log.info("Uninstalling add-on as part of wipe: " + addon.id); - Utils.catch.call(this, () => addon.uninstall())(); + Utils.catch(addon.uninstall)(); } }, @@ -538,22 +513,16 @@ AddonsStore.prototype = { * * @param addon * Addon instance - * @param ignoreRepoCheck - * Should we skip checking the Addons repository (primarially useful - * for testing and validation). * @return Boolean indicating whether it is appropriate for Sync */ - isAddonSyncable: function isAddonSyncable(addon, ignoreRepoCheck = false) { + isAddonSyncable: function isAddonSyncable(addon) { // Currently, we limit syncable add-ons to those that are: // 1) In a well-defined set of types // 2) Installed in the current profile // 3) Not installed by a foreign entity (i.e. installed by the app) // since they act like global extensions. // 4) Is not a hotfix. - // 5) The addons XPIProvider doesn't veto it (i.e not being installed in - // the profile directory, or any other reasons it says the addon can't - // be synced) - // 6) Are installed from AMO + // 5) Are installed from AMO // We could represent the test as a complex boolean expression. We go the // verbose route so the failure reason is logged. @@ -573,12 +542,6 @@ AddonsStore.prototype = { return false; } - // If the addon manager says it's not syncable, we skip it. - if (!addon.isSyncable) { - this._log.debug(addon.id + " not syncable: vetoed by the addon manager."); - return false; - } - // This may be too aggressive. If an add-on is downloaded from AMO and // manually placed in the profile directory, foreignInstall will be set. // Arguably, that add-on should be syncable. @@ -589,20 +552,15 @@ AddonsStore.prototype = { } // Ignore hotfix extensions (bug 741670). The pref may not be defined. - // XXX - note that addon.isSyncable will be false for hotfix addons, so - // this check isn't strictly necessary - except for Sync tests which aren't - // setup to create a "real" hotfix addon. This can be removed once those - // tests are fixed (but keeping it doesn't hurt either) if (this._extensionsPrefs.get("hotfix.id", null) == addon.id) { this._log.debug(addon.id + " not syncable: is a hotfix."); return false; } - // If the AddonRepository's cache isn't enabled (which it typically isn't - // in tests), getCachedAddonByID always returns null - so skip the check - // in that case. We also provide a way to specifically opt-out of the check - // even if the cache is enabled, which is used by the validators. - if (ignoreRepoCheck || !AddonRepository.cacheEnabled) { + // We provide a back door to skip the repository checking of an add-on. + // This is utilized by the tests to make testing easier. Users could enable + // this, but it would sacrifice security. + if (Svc.Prefs.get("addons.ignoreRepositoryChecking", false)) { return true; } @@ -745,69 +703,3 @@ AddonsTracker.prototype = { this.reconciler.stopListening(); }, }; - -class AddonValidator extends CollectionValidator { - constructor(engine = null) { - super("addons", "id", [ - "addonID", - "enabled", - "applicationID", - "source" - ]); - this.engine = engine; - } - - getClientItems() { - return Promise.all([ - new Promise(resolve => - AddonManager.getAllAddons(resolve)), - new Promise(resolve => - AddonManager.getAddonsWithOperationsByTypes(["extension", "theme"], resolve)), - ]).then(([installed, addonsWithPendingOperation]) => { - // Addons pending install won't be in the first list, but addons pending - // uninstall/enable/disable will be in both lists. - let all = new Map(installed.map(addon => [addon.id, addon])); - for (let addon of addonsWithPendingOperation) { - all.set(addon.id, addon); - } - // Convert to an array since Map.prototype.values returns an iterable - return [...all.values()]; - }); - } - - normalizeClientItem(item) { - let enabled = !item.userDisabled; - if (item.pendingOperations & AddonManager.PENDING_ENABLE) { - enabled = true; - } else if (item.pendingOperations & AddonManager.PENDING_DISABLE) { - enabled = false; - } - return { - enabled, - id: item.syncGUID, - addonID: item.id, - applicationID: Services.appinfo.ID, - source: "amo", // check item.foreignInstall? - original: item - }; - } - - normalizeServerItem(item) { - let guid = this.engine._findDupe(item); - if (guid) { - item.id = guid; - } - return item; - } - - clientUnderstands(item) { - return item.applicationID === Services.appinfo.ID; - } - - syncedByClient(item) { - return !item.original.hidden && - !item.original.isSystem && - !(item.original.pendingOperations & AddonManager.PENDING_UNINSTALL) && - this.engine.isAddonSyncable(item.original, true); - } -} diff --git a/services/sync/modules/engines/bookmarks.js b/services/sync/modules/engines/bookmarks.js index 76a198a8b..1936afc3f 100644 --- a/services/sync/modules/engines/bookmarks.js +++ b/services/sync/modules/engines/bookmarks.js @@ -6,12 +6,11 @@ this.EXPORTED_SYMBOLS = ['BookmarksEngine', "PlacesItem", "Bookmark", "BookmarkFolder", "BookmarkQuery", "Livemark", "BookmarkSeparator"]; -var Cc = Components.classes; -var Ci = Components.interfaces; -var Cu = Components.utils; +const Cc = Components.classes; +const Ci = Components.interfaces; +const Cu = Components.utils; Cu.import("resource://gre/modules/PlacesUtils.jsm"); -Cu.import("resource://gre/modules/PlacesSyncUtils.jsm"); Cu.import("resource://gre/modules/XPCOMUtils.jsm"); Cu.import("resource://services-common/async.js"); Cu.import("resource://services-sync/constants.js"); @@ -20,57 +19,21 @@ Cu.import("resource://services-sync/record.js"); Cu.import("resource://services-sync/util.js"); Cu.import("resource://gre/modules/Task.jsm"); Cu.import("resource://gre/modules/PlacesBackups.jsm"); -XPCOMUtils.defineLazyModuleGetter(this, "BookmarkValidator", - "resource://services-sync/bookmark_validator.js"); -XPCOMUtils.defineLazyGetter(this, "PlacesBundle", () => { - let bundleService = Cc["@mozilla.org/intl/stringbundle;1"] - .getService(Ci.nsIStringBundleService); - return bundleService.createBundle("chrome://places/locale/places.properties"); -}); - -const ANNOS_TO_TRACK = [PlacesSyncUtils.bookmarks.DESCRIPTION_ANNO, - PlacesSyncUtils.bookmarks.SIDEBAR_ANNO, + +const ALLBOOKMARKS_ANNO = "AllBookmarks"; +const DESCRIPTION_ANNO = "bookmarkProperties/description"; +const SIDEBAR_ANNO = "bookmarkProperties/loadInSidebar"; +const MOBILEROOT_ANNO = "mobile/bookmarksRoot"; +const MOBILE_ANNO = "MobileBookmarks"; +const EXCLUDEBACKUP_ANNO = "places/excludeFromBackup"; +const SMART_BOOKMARKS_ANNO = "Places/SmartBookmark"; +const PARENT_ANNO = "sync/parent"; +const ORGANIZERQUERY_ANNO = "PlacesOrganizer/OrganizerQuery"; +const ANNOS_TO_TRACK = [DESCRIPTION_ANNO, SIDEBAR_ANNO, PlacesUtils.LMANNO_FEEDURI, PlacesUtils.LMANNO_SITEURI]; const SERVICE_NOT_SUPPORTED = "Service not supported on this platform"; const FOLDER_SORTINDEX = 1000000; -const { - SOURCE_SYNC, - SOURCE_IMPORT, - SOURCE_IMPORT_REPLACE, -} = Ci.nsINavBookmarksService; - -const SQLITE_MAX_VARIABLE_NUMBER = 999; - -const ORGANIZERQUERY_ANNO = "PlacesOrganizer/OrganizerQuery"; -const ALLBOOKMARKS_ANNO = "AllBookmarks"; -const MOBILE_ANNO = "MobileBookmarks"; - -// The tracker ignores changes made by bookmark import and restore, and -// changes made by Sync. We don't need to exclude `SOURCE_IMPORT`, but both -// import and restore fire `bookmarks-restore-*` observer notifications, and -// the tracker doesn't currently distinguish between the two. -const IGNORED_SOURCES = [SOURCE_SYNC, SOURCE_IMPORT, SOURCE_IMPORT_REPLACE]; - -// Returns the constructor for a bookmark record type. -function getTypeObject(type) { - switch (type) { - case "bookmark": - case "microsummary": - return Bookmark; - case "query": - return BookmarkQuery; - case "folder": - return BookmarkFolder; - case "livemark": - return Livemark; - case "separator": - return BookmarkSeparator; - case "item": - return PlacesItem; - } - return null; -} this.PlacesItem = function PlacesItem(collection, id, type) { CryptoWrapper.call(this, collection, id); @@ -89,32 +52,26 @@ PlacesItem.prototype = { }, getTypeObject: function PlacesItem_getTypeObject(type) { - let recordObj = getTypeObject(type); - if (!recordObj) { - throw new Error("Unknown places item object type: " + type); + switch (type) { + case "bookmark": + case "microsummary": + return Bookmark; + case "query": + return BookmarkQuery; + case "folder": + return BookmarkFolder; + case "livemark": + return Livemark; + case "separator": + return BookmarkSeparator; + case "item": + return PlacesItem; } - return recordObj; + throw "Unknown places item object type: " + type; }, __proto__: CryptoWrapper.prototype, _logName: "Sync.Record.PlacesItem", - - // Converts the record to a Sync bookmark object that can be passed to - // `PlacesSyncUtils.bookmarks.{insert, update}`. - toSyncBookmark() { - return { - kind: this.type, - syncId: this.id, - parentSyncId: this.parentid, - }; - }, - - // Populates the record from a Sync bookmark object returned from - // `PlacesSyncUtils.bookmarks.fetch`. - fromSyncBookmark(item) { - this.parentid = item.parentSyncId; - this.parentName = item.parentTitle; - }, }; Utils.deferGetSet(PlacesItem, @@ -127,27 +84,6 @@ this.Bookmark = function Bookmark(collection, id, type) { Bookmark.prototype = { __proto__: PlacesItem.prototype, _logName: "Sync.Record.Bookmark", - - toSyncBookmark() { - let info = PlacesItem.prototype.toSyncBookmark.call(this); - info.title = this.title; - info.url = this.bmkUri; - info.description = this.description; - info.loadInSidebar = this.loadInSidebar; - info.tags = this.tags; - info.keyword = this.keyword; - return info; - }, - - fromSyncBookmark(item) { - PlacesItem.prototype.fromSyncBookmark.call(this, item); - this.title = item.title; - this.bmkUri = item.url.href; - this.description = item.description; - this.loadInSidebar = item.loadInSidebar; - this.tags = item.tags; - this.keyword = item.keyword; - }, }; Utils.deferGetSet(Bookmark, @@ -161,19 +97,6 @@ this.BookmarkQuery = function BookmarkQuery(collection, id) { BookmarkQuery.prototype = { __proto__: Bookmark.prototype, _logName: "Sync.Record.BookmarkQuery", - - toSyncBookmark() { - let info = Bookmark.prototype.toSyncBookmark.call(this); - info.folder = this.folderName; - info.query = this.queryId; - return info; - }, - - fromSyncBookmark(item) { - Bookmark.prototype.fromSyncBookmark.call(this, item); - this.folderName = item.folder; - this.queryId = item.query; - }, }; Utils.deferGetSet(BookmarkQuery, @@ -186,20 +109,6 @@ this.BookmarkFolder = function BookmarkFolder(collection, id, type) { BookmarkFolder.prototype = { __proto__: PlacesItem.prototype, _logName: "Sync.Record.Folder", - - toSyncBookmark() { - let info = PlacesItem.prototype.toSyncBookmark.call(this); - info.description = this.description; - info.title = this.title; - return info; - }, - - fromSyncBookmark(item) { - PlacesItem.prototype.fromSyncBookmark.call(this, item); - this.title = item.title; - this.description = item.description; - this.children = item.childSyncIds; - }, }; Utils.deferGetSet(BookmarkFolder, "cleartext", ["description", "title", @@ -211,21 +120,6 @@ this.Livemark = function Livemark(collection, id) { Livemark.prototype = { __proto__: BookmarkFolder.prototype, _logName: "Sync.Record.Livemark", - - toSyncBookmark() { - let info = BookmarkFolder.prototype.toSyncBookmark.call(this); - info.feed = this.feedUri; - info.site = this.siteUri; - return info; - }, - - fromSyncBookmark(item) { - BookmarkFolder.prototype.fromSyncBookmark.call(this, item); - this.feedUri = item.feed.href; - if (item.site) { - this.siteUri = item.site.href; - } - }, }; Utils.deferGetSet(Livemark, "cleartext", ["siteUri", "feedUri"]); @@ -236,15 +130,81 @@ this.BookmarkSeparator = function BookmarkSeparator(collection, id) { BookmarkSeparator.prototype = { __proto__: PlacesItem.prototype, _logName: "Sync.Record.Separator", - - fromSyncBookmark(item) { - PlacesItem.prototype.fromSyncBookmark.call(this, item); - this.pos = item.index; - }, }; Utils.deferGetSet(BookmarkSeparator, "cleartext", "pos"); + +let kSpecialIds = { + + // Special IDs. Note that mobile can attempt to create a record on + // dereference; special accessors are provided to prevent recursion within + // observers. + guids: ["menu", "places", "tags", "toolbar", "unfiled", "mobile"], + + // Create the special mobile folder to store mobile bookmarks. + createMobileRoot: function createMobileRoot() { + let root = PlacesUtils.placesRootId; + let mRoot = PlacesUtils.bookmarks.createFolder(root, "mobile", -1); + PlacesUtils.annotations.setItemAnnotation( + mRoot, MOBILEROOT_ANNO, 1, 0, PlacesUtils.annotations.EXPIRE_NEVER); + PlacesUtils.annotations.setItemAnnotation( + mRoot, EXCLUDEBACKUP_ANNO, 1, 0, PlacesUtils.annotations.EXPIRE_NEVER); + return mRoot; + }, + + findMobileRoot: function findMobileRoot(create) { + // Use the (one) mobile root if it already exists. + let root = PlacesUtils.annotations.getItemsWithAnnotation(MOBILEROOT_ANNO, {}); + if (root.length != 0) + return root[0]; + + if (create) + return this.createMobileRoot(); + + return null; + }, + + // Accessors for IDs. + isSpecialGUID: function isSpecialGUID(g) { + return this.guids.indexOf(g) != -1; + }, + + specialIdForGUID: function specialIdForGUID(guid, create) { + if (guid == "mobile") { + return this.findMobileRoot(create); + } + return this[guid]; + }, + + // Don't bother creating mobile: if it doesn't exist, this ID can't be it! + specialGUIDForId: function specialGUIDForId(id) { + for each (let guid in this.guids) + if (this.specialIdForGUID(guid, false) == id) + return guid; + return null; + }, + + get menu() { + return PlacesUtils.bookmarksMenuFolderId; + }, + get places() { + return PlacesUtils.placesRootId; + }, + get tags() { + return PlacesUtils.tagsFolderId; + }, + get toolbar() { + return PlacesUtils.toolbarFolderId; + }, + get unfiled() { + return PlacesUtils.unfiledBookmarksFolderId; + }, + get mobile() { + return this.findMobileRoot(true); + }, +}; + this.BookmarksEngine = function BookmarksEngine(service) { SyncEngine.call(this, "Bookmarks", service); } @@ -257,103 +217,68 @@ BookmarksEngine.prototype = { _defaultSort: "index", syncPriority: 4, - allowSkippedRecord: false, - - // A diagnostic helper to get the string value for a bookmark's URL given - // its ID. Always returns a string - on error will return a string in the - // form of "<description of error>" as this is purely for, eg, logging. - // (This means hitting the DB directly and we don't bother using a cached - // statement - we should rarely hit this.) - _getStringUrlForId(id) { - let url; - try { - let stmt = this._store._getStmt(` - SELECT h.url - FROM moz_places h - JOIN moz_bookmarks b ON h.id = b.fk - WHERE b.id = :id`); - stmt.params.id = id; - let rows = Async.querySpinningly(stmt, ["url"]); - url = rows.length == 0 ? "<not found>" : rows[0].url; - } catch (ex) { - if (Async.isShutdownException(ex)) { - throw ex; - } - if (ex instanceof Ci.mozIStorageError) { - url = `<failed: Storage error: ${ex.message} (${ex.result})>`; - } else { - url = `<failed: ${ex.toString()}>`; - } - } - return url; - }, - _guidMapFailed: false, - _buildGUIDMap: function _buildGUIDMap() { - let store = this._store; - let guidMap = {}; - let tree = Async.promiseSpinningly(PlacesUtils.promiseBookmarksTree("", { - includeItemIds: true - })); - function* walkBookmarksTree(tree, parent=null) { - if (tree) { - // Skip root node - if (parent) { - yield [tree, parent]; + _sync: function _sync() { + let engine = this; + let batchEx = null; + + // Try running sync in batch mode + PlacesUtils.bookmarks.runInBatchMode({ + runBatched: function wrappedSync() { + try { + SyncEngine.prototype._sync.call(engine); } - if (tree.children) { - for (let child of tree.children) { - store._sleep(0); // avoid jank while looping. - yield* walkBookmarksTree(child, tree); - } + catch(ex) { + batchEx = ex; } } - } + }, null); - function* walkBookmarksRoots(tree, rootIDs) { - for (let id of rootIDs) { - let bookmarkRoot = tree.children.find(child => child.id === id); - if (bookmarkRoot === null) { - continue; - } - yield* walkBookmarksTree(bookmarkRoot, tree); - } + // Expose the exception if something inside the batch failed + if (batchEx != null) { + throw batchEx; } + }, - let rootsToWalk = getChangeRootIds(); - - for (let [node, parent] of walkBookmarksRoots(tree, rootsToWalk)) { - let {guid, id, type: placeType} = node; - guid = PlacesSyncUtils.bookmarks.guidToSyncId(guid); + _guidMapFailed: false, + _buildGUIDMap: function _buildGUIDMap() { + let guidMap = {}; + for (let guid in this._store.getAllIDs()) { + // Figure out with which key to store the mapping. let key; - switch (placeType) { - case PlacesUtils.TYPE_X_MOZ_PLACE: - // Bookmark - let query = null; - if (node.annos && node.uri.startsWith("place:")) { - query = node.annos.find(({name}) => - name === PlacesSyncUtils.bookmarks.SMART_BOOKMARKS_ANNO); - } - if (query && query.value) { - key = "q" + query.value; - } else { - key = "b" + node.uri + ":" + (node.title || ""); - } + let id = this._store.idForGUID(guid); + switch (PlacesUtils.bookmarks.getItemType(id)) { + case PlacesUtils.bookmarks.TYPE_BOOKMARK: + + // Smart bookmarks map to their annotation value. + let queryId; + try { + queryId = PlacesUtils.annotations.getItemAnnotation( + id, SMART_BOOKMARKS_ANNO); + } catch(ex) {} + + if (queryId) + key = "q" + queryId; + else + key = "b" + PlacesUtils.bookmarks.getBookmarkURI(id).spec + ":" + + PlacesUtils.bookmarks.getItemTitle(id); break; - case PlacesUtils.TYPE_X_MOZ_PLACE_CONTAINER: - // Folder - key = "f" + (node.title || ""); + case PlacesUtils.bookmarks.TYPE_FOLDER: + key = "f" + PlacesUtils.bookmarks.getItemTitle(id); break; - case PlacesUtils.TYPE_X_MOZ_PLACE_SEPARATOR: - // Separator - key = "s" + node.index; + case PlacesUtils.bookmarks.TYPE_SEPARATOR: + key = "s" + PlacesUtils.bookmarks.getItemIndex(id); break; default: - this._log.error("Unknown place type: '"+placeType+"'"); continue; } - let parentName = parent.title || ""; + // The mapping is on a per parent-folder-name basis. + let parent = PlacesUtils.bookmarks.getFolderIdForItem(id); + if (parent <= 0) + continue; + + let parentName = PlacesUtils.bookmarks.getItemTitle(parent); if (guidMap[parentName] == null) guidMap[parentName] = {}; @@ -381,17 +306,17 @@ BookmarksEngine.prototype = { // hack should get them to dupe correctly. if (item.queryId) { key = "q" + item.queryId; - altKey = "b" + item.bmkUri + ":" + (item.title || ""); + altKey = "b" + item.bmkUri + ":" + item.title; break; } // No queryID? Fall through to the regular bookmark case. case "bookmark": case "microsummary": - key = "b" + item.bmkUri + ":" + (item.title || ""); + key = "b" + item.bmkUri + ":" + item.title; break; case "folder": case "livemark": - key = "f" + (item.title || ""); + key = "f" + item.title; break; case "separator": key = "s" + item.pos; @@ -405,22 +330,21 @@ BookmarksEngine.prototype = { let guidMap = this._guidMap; // Give the GUID if we have the matching pair. - let parentName = item.parentName || ""; - this._log.trace("Finding mapping: " + parentName + ", " + key); - let parent = guidMap[parentName]; - + this._log.trace("Finding mapping: " + item.parentName + ", " + key); + let parent = guidMap[item.parentName]; + if (!parent) { this._log.trace("No parent => no dupe."); return undefined; } - + let dupe = parent[key]; - + if (dupe) { this._log.trace("Mapped dupe: " + dupe); return dupe; } - + if (altKey) { dupe = parent[altKey]; if (dupe) { @@ -428,7 +352,7 @@ BookmarksEngine.prototype = { return dupe; } } - + this._log.trace("No dupe found for key " + key + "/" + altKey + "."); return undefined; }, @@ -437,7 +361,7 @@ BookmarksEngine.prototype = { SyncEngine.prototype._syncStartup.call(this); let cb = Async.makeSpinningCallback(); - Task.spawn(function* () { + Task.spawn(function() { // For first-syncs, make a backup for the user to restore if (this.lastSync == 0) { this._log.debug("Bookmarks backup starting."); @@ -449,7 +373,8 @@ BookmarksEngine.prototype = { // Failure to create a backup is somewhat bad, but probably not bad // enough to prevent syncing of bookmarks - so just log the error and // continue. - this._log.warn("Error while backing up bookmarks, but continuing with sync", ex); + this._log.warn("Got exception \"" + Utils.exceptionStr(ex) + + "\" backing up bookmarks, but continuing with sync."); cb(); } ); @@ -464,10 +389,9 @@ BookmarksEngine.prototype = { try { guidMap = this._buildGUIDMap(); } catch (ex) { - if (Async.isShutdownException(ex)) { - throw ex; - } - this._log.warn("Error while building GUID map, skipping all other incoming items", ex); + this._log.warn("Got exception \"" + Utils.exceptionStr(ex) + + "\" building GUID map." + + " Skipping all other incoming items."); throw {code: Engine.prototype.eEngineAbortApplyIncoming, cause: ex}; } @@ -476,71 +400,17 @@ BookmarksEngine.prototype = { }); this._store._childrenToOrder = {}; - this._store.clearPendingDeletions(); - }, - - _deletePending() { - // Delete pending items -- See the comment above BookmarkStore's deletePending - let newlyModified = Async.promiseSpinningly(this._store.deletePending()); - let now = this._tracker._now(); - this._log.debug("Deleted pending items", newlyModified); - for (let modifiedSyncID of newlyModified) { - if (!this._modified.has(modifiedSyncID)) { - this._modified.set(modifiedSyncID, { timestamp: now, deleted: false }); - } - } - }, - - // We avoid reviving folders since reviving them properly would require - // reviving their children as well. Unfortunately, this is the wrong choice - // in the case of a bookmark restore where wipeServer failed -- if the - // server has the folder as deleted, we *would* want to reupload this folder. - // This is mitigated by the fact that we move any undeleted children to the - // grandparent when deleting the parent. - _shouldReviveRemotelyDeletedRecord(item) { - let kind = Async.promiseSpinningly( - PlacesSyncUtils.bookmarks.getKindForSyncId(item.id)); - if (kind === PlacesSyncUtils.bookmarks.KINDS.FOLDER) { - return false; - } - - // In addition to preventing the deletion of this record (handled by the caller), - // we need to mark the parent of this record for uploading next sync, in order - // to ensure its children array is accurate. - let modifiedTimestamp = this._modified.getModifiedTimestamp(item.id); - if (!modifiedTimestamp) { - // We only expect this to be called with items locally modified, so - // something strange is going on - play it safe and don't revive it. - this._log.error("_shouldReviveRemotelyDeletedRecord called on unmodified item: " + item.id); - return false; - } - - let localID = this._store.idForGUID(item.id); - let localParentID = PlacesUtils.bookmarks.getFolderIdForItem(localID); - let localParentSyncID = this._store.GUIDForId(localParentID); - - this._log.trace(`Reviving item "${item.id}" and marking parent ${localParentSyncID} as modified.`); - - if (!this._modified.has(localParentSyncID)) { - this._modified.set(localParentSyncID, { - timestamp: modifiedTimestamp, - deleted: false - }); - } - return true }, _processIncoming: function (newitems) { try { SyncEngine.prototype._processIncoming.call(this, newitems); } finally { - try { - this._deletePending(); - } finally { - // Reorder children. - this._store._orderChildren(); - delete this._store._childrenToOrder; - } + // Reorder children. + this._tracker.ignoreAll = true; + this._store._orderChildren(); + this._tracker.ignoreAll = false; + delete this._store._childrenToOrder; } }, @@ -575,154 +445,16 @@ BookmarksEngine.prototype = { } let mapped = this._mapDupe(item); this._log.debug(item.id + " mapped to " + mapped); - // We must return a string, not an object, and the entries in the GUIDMap - // are created via "new String()" making them an object. - return mapped ? mapped.toString() : mapped; - }, - - pullAllChanges() { - return new BookmarksChangeset(this._store.getAllIDs()); - }, - - pullNewChanges() { - let modifiedGUIDs = this._getModifiedGUIDs(); - if (!modifiedGUIDs.length) { - return new BookmarksChangeset(this._tracker.changedIDs); - } - - // We don't use `PlacesUtils.promiseDBConnection` here because - // `getChangedIDs` might be called while we're in a batch, meaning we - // won't see any changes until the batch finishes and the transaction - // commits. - let db = PlacesUtils.history.QueryInterface(Ci.nsPIPlacesDatabase) - .DBConnection; - - // Filter out tags, organizer queries, and other descendants that we're - // not tracking. We chunk `modifiedGUIDs` because SQLite limits the number - // of bound parameters per query. - for (let startIndex = 0; - startIndex < modifiedGUIDs.length; - startIndex += SQLITE_MAX_VARIABLE_NUMBER) { - - let chunkLength = Math.min(SQLITE_MAX_VARIABLE_NUMBER, - modifiedGUIDs.length - startIndex); - - let query = ` - WITH RECURSIVE - modifiedGuids(guid) AS ( - VALUES ${new Array(chunkLength).fill("(?)").join(", ")} - ), - syncedItems(id) AS ( - VALUES ${getChangeRootIds().map(id => `(${id})`).join(", ")} - UNION ALL - SELECT b.id - FROM moz_bookmarks b - JOIN syncedItems s ON b.parent = s.id - ) - SELECT b.guid - FROM modifiedGuids m - JOIN moz_bookmarks b ON b.guid = m.guid - LEFT JOIN syncedItems s ON b.id = s.id - WHERE s.id IS NULL - `; - - let statement = db.createAsyncStatement(query); - try { - for (let i = 0; i < chunkLength; i++) { - statement.bindByIndex(i, modifiedGUIDs[startIndex + i]); - } - let results = Async.querySpinningly(statement, ["guid"]); - for (let { guid } of results) { - let syncID = PlacesSyncUtils.bookmarks.guidToSyncId(guid); - this._tracker.removeChangedID(syncID); - } - } finally { - statement.finalize(); - } - } - - return new BookmarksChangeset(this._tracker.changedIDs); - }, - - // Returns an array of Places GUIDs for all changed items. Ignores deletions, - // which won't exist in the DB and shouldn't be removed from the tracker. - _getModifiedGUIDs() { - let guids = []; - for (let syncID in this._tracker.changedIDs) { - if (this._tracker.changedIDs[syncID].deleted === true) { - // The `===` check also filters out old persisted timestamps, - // which won't have a `deleted` property. - continue; - } - let guid = PlacesSyncUtils.bookmarks.syncIdToGuid(syncID); - guids.push(guid); - } - return guids; - }, - - // Called when _findDupe returns a dupe item and the engine has decided to - // switch the existing item to the new incoming item. - _switchItemToDupe(localDupeGUID, incomingItem) { - // We unconditionally change the item's ID in case the engine knows of - // an item but doesn't expose it through itemExists. If the API - // contract were stronger, this could be changed. - this._log.debug("Switching local ID to incoming: " + localDupeGUID + " -> " + - incomingItem.id); - this._store.changeItemID(localDupeGUID, incomingItem.id); - - // And mark the parent as being modified. Given we de-dupe based on the - // parent *name* it's possible the item having its GUID changed has a - // different parent from the incoming record. - // So we need to find the GUID of the local parent. - let now = this._tracker._now(); - let localID = this._store.idForGUID(incomingItem.id); - let localParentID = PlacesUtils.bookmarks.getFolderIdForItem(localID); - let localParentGUID = this._store.GUIDForId(localParentID); - this._modified.set(localParentGUID, { modified: now, deleted: false }); - - // And we also add the parent as reflected in the incoming record as the - // de-dupe process might have used an existing item in a different folder. - // But only if the parent exists, otherwise we will upload a deleted item - // when it might actually be valid, just unknown to us. Note that this - // scenario will still leave us with inconsistent client and server states; - // the incoming record on the server references a parent that isn't the - // actual parent locally - see bug 1297955. - if (localParentGUID != incomingItem.parentid) { - let remoteParentID = this._store.idForGUID(incomingItem.parentid); - if (remoteParentID > 0) { - // The parent specified in the record does exist, so we are going to - // attempt a move when we come to applying the record. Mark the parent - // as being modified so we will later upload it with the new child - // reference. - this._modified.set(incomingItem.parentid, { modified: now, deleted: false }); - } else { - // We aren't going to do a move as we don't have the parent (yet?). - // When applying the record we will add our special PARENT_ANNO - // annotation, so if it arrives in the future (either this Sync or a - // later one) it will be reparented. - this._log.debug(`Incoming duplicate item ${incomingItem.id} specifies ` + - `non-existing parent ${incomingItem.parentid}`); - } - } - - // The local, duplicate ID is always deleted on the server - but for - // bookmarks it is a logical delete. - // Simply adding this (now non-existing) ID to the tracker is enough. - this._modified.set(localDupeGUID, { modified: now, deleted: true }); - }, - getValidator() { - return new BookmarkValidator(); + return mapped; } }; function BookmarksStore(name, engine) { Store.call(this, name, engine); - this._foldersToDelete = new Set(); - this._atomsToDelete = new Set(); + // Explicitly nullify our references to our cached services so we don't leak Svc.Obs.add("places-shutdown", function() { - for (let query in this._stmts) { - let stmt = this._stmts[query]; + for each (let [query, stmt] in Iterator(this._stmts)) { stmt.finalize(); } this._stmts = {}; @@ -732,12 +464,70 @@ BookmarksStore.prototype = { __proto__: Store.prototype, itemExists: function BStore_itemExists(id) { - return this.idForGUID(id) > 0; + return this.idForGUID(id, true) > 0; }, + + /* + * If the record is a tag query, rewrite it to refer to the local tag ID. + * + * Otherwise, just return. + */ + preprocessTagQuery: function preprocessTagQuery(record) { + if (record.type != "query" || + record.bmkUri == null || + !record.folderName) + return; + + // Yes, this works without chopping off the "place:" prefix. + let uri = record.bmkUri + let queriesRef = {}; + let queryCountRef = {}; + let optionsRef = {}; + PlacesUtils.history.queryStringToQueries(uri, queriesRef, queryCountRef, + optionsRef); + + // We only process tag URIs. + if (optionsRef.value.resultType != optionsRef.value.RESULTS_AS_TAG_CONTENTS) + return; + + // Tag something to ensure that the tag exists. + let tag = record.folderName; + let dummyURI = Utils.makeURI("about:weave#BStore_preprocess"); + PlacesUtils.tagging.tagURI(dummyURI, [tag]); + + // Look for the id of the tag, which might just have been added. + let tags = this._getNode(PlacesUtils.tagsFolderId); + if (!(tags instanceof Ci.nsINavHistoryQueryResultNode)) { + this._log.debug("tags isn't an nsINavHistoryQueryResultNode; aborting."); + return; + } + tags.containerOpen = true; + try { + for (let i = 0; i < tags.childCount; i++) { + let child = tags.getChild(i); + if (child.title == tag) { + // Found the tag, so fix up the query to use the right id. + this._log.debug("Tag query folder: " + tag + " = " + child.itemId); + + this._log.trace("Replacing folders in: " + uri); + for each (let q in queriesRef.value) + q.setFolders([child.itemId], 1); + + record.bmkUri = PlacesUtils.history.queriesToQueryString( + queriesRef.value, queryCountRef.value, optionsRef.value); + return; + } + } + } + finally { + tags.containerOpen = false; + } + }, + applyIncoming: function BStore_applyIncoming(record) { this._log.debug("Applying record " + record.id); - let isSpecial = PlacesSyncUtils.bookmarks.ROOTS.includes(record.id); + let isSpecial = record.id in kSpecialIds; if (record.deleted) { if (isSpecial) { @@ -765,217 +555,548 @@ BookmarksStore.prototype = { return; } + // Preprocess the record before doing the normal apply. + this.preprocessTagQuery(record); + // Figure out the local id of the parent GUID if available let parentGUID = record.parentid; if (!parentGUID) { throw "Record " + record.id + " has invalid parentid: " + parentGUID; } - this._log.debug("Remote parent is " + parentGUID); + this._log.debug("Local parent is " + parentGUID); + + let parentId = this.idForGUID(parentGUID); + if (parentId > 0) { + // Save the parent id for modifying the bookmark later + record._parent = parentId; + record._orphan = false; + this._log.debug("Record " + record.id + " is not an orphan."); + } else { + this._log.trace("Record " + record.id + + " is an orphan: could not find parent " + parentGUID); + record._orphan = true; + } // Do the normal processing of incoming records Store.prototype.applyIncoming.call(this, record); - if (record.type == "folder" && record.children) { - this._childrenToOrder[record.id] = record.children; + // Do some post-processing if we have an item + let itemId = this.idForGUID(record.id); + if (itemId > 0) { + // Move any children that are looking for this folder as a parent + if (record.type == "folder") { + this._reparentOrphans(itemId); + // Reorder children later + if (record.children) + this._childrenToOrder[record.id] = record.children; + } + + // Create an annotation to remember that it needs reparenting. + if (record._orphan) { + PlacesUtils.annotations.setItemAnnotation( + itemId, PARENT_ANNO, parentGUID, 0, + PlacesUtils.annotations.EXPIRE_NEVER); + } + } + }, + + /** + * Find all ids of items that have a given value for an annotation + */ + _findAnnoItems: function BStore__findAnnoItems(anno, val) { + return PlacesUtils.annotations.getItemsWithAnnotation(anno, {}) + .filter(function(id) { + return PlacesUtils.annotations.getItemAnnotation(id, anno) == val; + }); + }, + + /** + * For the provided parent item, attach its children to it + */ + _reparentOrphans: function _reparentOrphans(parentId) { + // Find orphans and reunite with this folder parent + let parentGUID = this.GUIDForId(parentId); + let orphans = this._findAnnoItems(PARENT_ANNO, parentGUID); + + this._log.debug("Reparenting orphans " + orphans + " to " + parentId); + orphans.forEach(function(orphan) { + // Move the orphan to the parent and drop the missing parent annotation + if (this._reparentItem(orphan, parentId)) { + PlacesUtils.annotations.removeItemAnnotation(orphan, PARENT_ANNO); + } + }, this); + }, + + _reparentItem: function _reparentItem(itemId, parentId) { + this._log.trace("Attempting to move item " + itemId + " to new parent " + + parentId); + try { + if (parentId > 0) { + PlacesUtils.bookmarks.moveItem(itemId, parentId, + PlacesUtils.bookmarks.DEFAULT_INDEX); + return true; + } + } catch(ex) { + this._log.debug("Failed to reparent item. " + Utils.exceptionStr(ex)); + } + return false; + }, + + // Turn a record's nsINavBookmarksService constant and other attributes into + // a granular type for comparison. + _recordType: function _recordType(itemId) { + let bms = PlacesUtils.bookmarks; + let type = bms.getItemType(itemId); + + switch (type) { + case bms.TYPE_FOLDER: + if (PlacesUtils.annotations + .itemHasAnnotation(itemId, PlacesUtils.LMANNO_FEEDURI)) { + return "livemark"; + } + return "folder"; + + case bms.TYPE_BOOKMARK: + let bmkUri = bms.getBookmarkURI(itemId).spec; + if (bmkUri.indexOf("place:") == 0) { + return "query"; + } + return "bookmark"; + + case bms.TYPE_SEPARATOR: + return "separator"; + + default: + return null; } }, create: function BStore_create(record) { - let info = record.toSyncBookmark(); - // This can throw if we're inserting an invalid or incomplete bookmark. - // That's fine; the exception will be caught by `applyIncomingBatch` - // without aborting further processing. - let item = Async.promiseSpinningly(PlacesSyncUtils.bookmarks.insert(info)); - if (item) { - this._log.debug(`Created ${item.kind} ${item.syncId} under ${ - item.parentSyncId}`, item); + // Default to unfiled if we don't have the parent yet. + + // Valid parent IDs are all positive integers. Other values -- undefined, + // null, -1 -- all compare false for > 0, so this catches them all. We + // don't just use <= without the !, because undefined and null compare + // false for that, too! + if (!(record._parent > 0)) { + this._log.debug("Parent is " + record._parent + "; reparenting to unfiled."); + record._parent = kSpecialIds.unfiled; + } + + let newId; + switch (record.type) { + case "bookmark": + case "query": + case "microsummary": { + let uri = Utils.makeURI(record.bmkUri); + newId = PlacesUtils.bookmarks.insertBookmark( + record._parent, uri, PlacesUtils.bookmarks.DEFAULT_INDEX, record.title); + this._log.debug("created bookmark " + newId + " under " + record._parent + + " as " + record.title + " " + record.bmkUri); + + // Smart bookmark annotations are strings. + if (record.queryId) { + PlacesUtils.annotations.setItemAnnotation( + newId, SMART_BOOKMARKS_ANNO, record.queryId, 0, + PlacesUtils.annotations.EXPIRE_NEVER); + } + + if (Array.isArray(record.tags)) { + this._tagURI(uri, record.tags); + } + PlacesUtils.bookmarks.setKeywordForBookmark(newId, record.keyword); + if (record.description) { + PlacesUtils.annotations.setItemAnnotation( + newId, DESCRIPTION_ANNO, record.description, 0, + PlacesUtils.annotations.EXPIRE_NEVER); + } + + if (record.loadInSidebar) { + PlacesUtils.annotations.setItemAnnotation( + newId, SIDEBAR_ANNO, true, 0, + PlacesUtils.annotations.EXPIRE_NEVER); + } + + } break; + case "folder": + newId = PlacesUtils.bookmarks.createFolder( + record._parent, record.title, PlacesUtils.bookmarks.DEFAULT_INDEX); + this._log.debug("created folder " + newId + " under " + record._parent + + " as " + record.title); + + if (record.description) { + PlacesUtils.annotations.setItemAnnotation( + newId, DESCRIPTION_ANNO, record.description, 0, + PlacesUtils.annotations.EXPIRE_NEVER); + } + + // record.children will be dealt with in _orderChildren. + break; + case "livemark": + let siteURI = null; + if (!record.feedUri) { + this._log.debug("No feed URI: skipping livemark record " + record.id); + return; + } + if (PlacesUtils.annotations + .itemHasAnnotation(record._parent, PlacesUtils.LMANNO_FEEDURI)) { + this._log.debug("Invalid parent: skipping livemark record " + record.id); + return; + } + + if (record.siteUri != null) + siteURI = Utils.makeURI(record.siteUri); + + // Until this engine can handle asynchronous error reporting, we need to + // detect errors on creation synchronously. + let spinningCb = Async.makeSpinningCallback(); + + let livemarkObj = {title: record.title, + parentId: record._parent, + index: PlacesUtils.bookmarks.DEFAULT_INDEX, + feedURI: Utils.makeURI(record.feedUri), + siteURI: siteURI, + guid: record.id}; + PlacesUtils.livemarks.addLivemark(livemarkObj).then( + aLivemark => { spinningCb(null, [Components.results.NS_OK, aLivemark]) }, + () => { spinningCb(null, [Components.results.NS_ERROR_UNEXPECTED, aLivemark]) } + ); + + let [status, livemark] = spinningCb.wait(); + if (!Components.isSuccessCode(status)) { + throw status; + } + + this._log.debug("Created livemark " + livemark.id + " under " + + livemark.parentId + " as " + livemark.title + + ", " + livemark.siteURI.spec + ", " + + livemark.feedURI.spec + ", GUID " + + livemark.guid); + break; + case "separator": + newId = PlacesUtils.bookmarks.insertSeparator( + record._parent, PlacesUtils.bookmarks.DEFAULT_INDEX); + this._log.debug("created separator " + newId + " under " + record._parent); + break; + case "item": + this._log.debug(" -> got a generic places item.. do nothing?"); + return; + default: + this._log.error("_create: Unknown item type: " + record.type); + return; + } + + if (newId) { + // Livemarks can set the GUID through the API, so there's no need to + // do that here. + this._log.trace("Setting GUID of new item " + newId + " to " + record.id); + this._setGUID(newId, record.id); + } + }, + + // Factored out of `remove` to avoid redundant DB queries when the Places ID + // is already known. + removeById: function removeById(itemId, guid) { + let type = PlacesUtils.bookmarks.getItemType(itemId); + + switch (type) { + case PlacesUtils.bookmarks.TYPE_BOOKMARK: + this._log.debug(" -> removing bookmark " + guid); + PlacesUtils.bookmarks.removeItem(itemId); + break; + case PlacesUtils.bookmarks.TYPE_FOLDER: + this._log.debug(" -> removing folder " + guid); + PlacesUtils.bookmarks.removeItem(itemId); + break; + case PlacesUtils.bookmarks.TYPE_SEPARATOR: + this._log.debug(" -> removing separator " + guid); + PlacesUtils.bookmarks.removeItem(itemId); + break; + default: + this._log.error("remove: Unknown item type: " + type); + break; } }, remove: function BStore_remove(record) { - if (PlacesSyncUtils.bookmarks.isRootSyncID(record.id)) { + if (kSpecialIds.isSpecialGUID(record.id)) { this._log.warn("Refusing to remove special folder " + record.id); return; } - let recordKind = Async.promiseSpinningly( - PlacesSyncUtils.bookmarks.getKindForSyncId(record.id)); - let isFolder = recordKind === PlacesSyncUtils.bookmarks.KINDS.FOLDER; - this._log.trace(`Buffering removal of item "${record.id}" of type "${recordKind}".`); - if (isFolder) { - this._foldersToDelete.add(record.id); - } else { - this._atomsToDelete.add(record.id); + + let itemId = this.idForGUID(record.id); + if (itemId <= 0) { + this._log.debug("Item " + record.id + " already removed"); + return; } + this.removeById(itemId, record.id); }, - update: function BStore_update(record) { - let info = record.toSyncBookmark(); - let item = Async.promiseSpinningly(PlacesSyncUtils.bookmarks.update(info)); - if (item) { - this._log.debug(`Updated ${item.kind} ${item.syncId} under ${ - item.parentSyncId}`, item); - } + _taggableTypes: ["bookmark", "microsummary", "query"], + isTaggable: function isTaggable(recordType) { + return this._taggableTypes.indexOf(recordType) != -1; }, - _orderChildren: function _orderChildren() { - let promises = Object.keys(this._childrenToOrder).map(syncID => { - let children = this._childrenToOrder[syncID]; - return PlacesSyncUtils.bookmarks.order(syncID, children).catch(ex => { - this._log.debug(`Could not order children for ${syncID}`, ex); - }); - }); - Async.promiseSpinningly(Promise.all(promises)); - }, - - // There's some complexity here around pending deletions. Our goals: - // - // - Don't delete any bookmarks a user has created but not explicitly deleted - // (This includes any bookmark that was not a child of the folder at the - // time the deletion was recorded, and also bookmarks restored from a backup). - // - Don't undelete any bookmark without ensuring the server structure - // includes it (see `BookmarkEngine.prototype._shouldReviveRemotelyDeletedRecord`) - // - // This leads the following approach: - // - // - Additions, moves, and updates are processed before deletions. - // - To do this, all deletion operations are buffered during a sync. Folders - // we plan on deleting have their sync id's stored in `this._foldersToDelete`, - // and non-folders we plan on deleting have their sync id's stored in - // `this._atomsToDelete`. - // - The exception to this is the moves that occur to fix the order of bookmark - // children, which are performed after we process deletions. - // - Non-folders are deleted before folder deletions, so that when we process - // folder deletions we know the correct state. - // - Remote deletions always win for folders, but do not result in recursive - // deletion of children. This is a hack because we're not able to distinguish - // between value changes and structural changes to folders, and we don't even - // have the old server record to compare to. See `BookmarkEngine`'s - // `_shouldReviveRemotelyDeletedRecord` method. - // - When a folder is deleted, its remaining children are moved in order to - // their closest living ancestor. If this is interrupted (unlikely, but - // possible given that we don't perform this operation in a transaction), - // we revive the folder. - // - Remote deletions can lose for non-folders, but only until we handle - // bookmark restores correctly (removing stale state from the server -- this - // is to say, if bug 1230011 is fixed, we should never revive bookmarks). - - deletePending: Task.async(function* deletePending() { - yield this._deletePendingAtoms(); - let guidsToUpdate = yield this._deletePendingFolders(); - this.clearPendingDeletions(); - return guidsToUpdate; - }), - - clearPendingDeletions() { - this._foldersToDelete.clear(); - this._atomsToDelete.clear(); - }, - - _deleteAtom: Task.async(function* _deleteAtom(syncID) { - try { - let info = yield PlacesSyncUtils.bookmarks.remove(syncID, { - preventRemovalOfNonEmptyFolders: true - }); - this._log.trace(`Removed item ${syncID} with type ${info.type}`); - } catch (ex) { - // Likely already removed. - this._log.trace(`Error removing ${syncID}`, ex); + update: function BStore_update(record) { + let itemId = this.idForGUID(record.id); + + if (itemId <= 0) { + this._log.debug("Skipping update for unknown item: " + record.id); + return; } - }), - - _deletePendingAtoms() { - return Promise.all( - [...this._atomsToDelete.values()] - .map(syncID => this._deleteAtom(syncID))); - }, - - // Returns an array of sync ids that need updates. - _deletePendingFolders: Task.async(function* _deletePendingFolders() { - // To avoid data loss, we don't want to just delete the folder outright, - // so we buffer folder deletions and process them at the end (now). - // - // At this point, any member in the folder that remains is either a folder - // pending deletion (which we'll get to in this function), or an item that - // should not be deleted. To avoid deleting these items, we first move them - // to the parent of the folder we're about to delete. - let needUpdate = new Set(); - for (let syncId of this._foldersToDelete) { - let childSyncIds = yield PlacesSyncUtils.bookmarks.fetchChildSyncIds(syncId); - if (!childSyncIds.length) { - // No children -- just delete the folder. - yield this._deleteAtom(syncId) - continue; - } - // We could avoid some redundant work here by finding the nearest - // grandparent who isn't present in `this._toDelete`... - let grandparentSyncId = this.GUIDForId( - PlacesUtils.bookmarks.getFolderIdForItem( - this.idForGUID(PlacesSyncUtils.bookmarks.syncIdToGuid(syncId)))); + // Two items are the same type if they have the same ItemType in Places, + // and also share some key characteristics (e.g., both being livemarks). + // We figure this out by examining the item to find the equivalent granular + // (string) type. + // If they're not the same type, we can't just update attributes. Delete + // then recreate the record instead. + let localItemType = this._recordType(itemId); + let remoteRecordType = record.type; + this._log.trace("Local type: " + localItemType + ". " + + "Remote type: " + remoteRecordType + "."); + + if (localItemType != remoteRecordType) { + this._log.debug("Local record and remote record differ in type. " + + "Deleting and recreating."); + this.removeById(itemId, record.id); + this.create(record); + return; + } - this._log.trace(`Moving ${childSyncIds.length} children of "${syncId}" to ` + - `grandparent "${grandparentSyncId}" before deletion.`); + this._log.trace("Updating " + record.id + " (" + itemId + ")"); - // Move children out of the parent and into the grandparent - yield Promise.all(childSyncIds.map(child => PlacesSyncUtils.bookmarks.update({ - syncId: child, - parentSyncId: grandparentSyncId - }))); + // Move the bookmark to a new parent or new position if necessary + if (record._parent > 0 && + PlacesUtils.bookmarks.getFolderIdForItem(itemId) != record._parent) { + this._reparentItem(itemId, record._parent); + } - // Delete the (now empty) parent - try { - yield PlacesSyncUtils.bookmarks.remove(syncId, { - preventRemovalOfNonEmptyFolders: true - }); - } catch (e) { - // We failed, probably because someone added something to this folder - // between when we got the children and now (or the database is corrupt, - // or something else happened...) This is unlikely, but possible. To - // avoid corruption in this case, we need to reupload the record to the - // server. - // - // (Ideally this whole operation would be done in a transaction, and this - // wouldn't be possible). - needUpdate.add(syncId); + for (let [key, val] in Iterator(record.cleartext)) { + switch (key) { + case "title": + PlacesUtils.bookmarks.setItemTitle(itemId, val); + break; + case "bmkUri": + PlacesUtils.bookmarks.changeBookmarkURI(itemId, Utils.makeURI(val)); + break; + case "tags": + if (Array.isArray(val)) { + if (this.isTaggable(remoteRecordType)) { + this._tagID(itemId, val); + } else { + this._log.debug("Remote record type is invalid for tags: " + remoteRecordType); + } + } + break; + case "keyword": + PlacesUtils.bookmarks.setKeywordForBookmark(itemId, val); + break; + case "description": + if (val) { + PlacesUtils.annotations.setItemAnnotation( + itemId, DESCRIPTION_ANNO, val, 0, + PlacesUtils.annotations.EXPIRE_NEVER); + } else { + PlacesUtils.annotations.removeItemAnnotation(itemId, DESCRIPTION_ANNO); + } + break; + case "loadInSidebar": + if (val) { + PlacesUtils.annotations.setItemAnnotation( + itemId, SIDEBAR_ANNO, true, 0, + PlacesUtils.annotations.EXPIRE_NEVER); + } else { + PlacesUtils.annotations.removeItemAnnotation(itemId, SIDEBAR_ANNO); + } + break; + case "queryId": + PlacesUtils.annotations.setItemAnnotation( + itemId, SMART_BOOKMARKS_ANNO, val, 0, + PlacesUtils.annotations.EXPIRE_NEVER); + break; } + } + }, - // Add children (for parentid) and grandparent (for children list) to set - // of records needing an update, *unless* they're marked for deletion. - if (!this._foldersToDelete.has(grandparentSyncId)) { - needUpdate.add(grandparentSyncId); - } - for (let childSyncId of childSyncIds) { - if (!this._foldersToDelete.has(childSyncId)) { - needUpdate.add(childSyncId); + _orderChildren: function _orderChildren() { + for (let [guid, children] in Iterator(this._childrenToOrder)) { + // Reorder children according to the GUID list. Gracefully deal + // with missing items, e.g. locally deleted. + let delta = 0; + let parent = null; + for (let idx = 0; idx < children.length; idx++) { + let itemid = this.idForGUID(children[idx]); + if (itemid == -1) { + delta += 1; + this._log.trace("Could not locate record " + children[idx]); + continue; + } + try { + // This code path could be optimized by caching the parent earlier. + // Doing so should take in count any edge case due to reparenting + // or parent invalidations though. + if (!parent) { + parent = PlacesUtils.bookmarks.getFolderIdForItem(itemid); + } + PlacesUtils.bookmarks.moveItem(itemid, parent, idx - delta); + } catch (ex) { + this._log.debug("Could not move item " + children[idx] + ": " + ex); } } } - return [...needUpdate]; - }), + }, changeItemID: function BStore_changeItemID(oldID, newID) { this._log.debug("Changing GUID " + oldID + " to " + newID); - Async.promiseSpinningly(PlacesSyncUtils.bookmarks.changeGuid(oldID, newID)); + // Make sure there's an item to change GUIDs + let itemId = this.idForGUID(oldID); + if (itemId <= 0) + return; + + this._setGUID(itemId, newID); + }, + + _getNode: function BStore__getNode(folder) { + let query = PlacesUtils.history.getNewQuery(); + query.setFolders([folder], 1); + return PlacesUtils.history.executeQuery( + query, PlacesUtils.history.getNewQueryOptions()).root; + }, + + _getTags: function BStore__getTags(uri) { + try { + if (typeof(uri) == "string") + uri = Utils.makeURI(uri); + } catch(e) { + this._log.warn("Could not parse URI \"" + uri + "\": " + e); + } + return PlacesUtils.tagging.getTagsForURI(uri, {}); + }, + + _getDescription: function BStore__getDescription(id) { + try { + return PlacesUtils.annotations.getItemAnnotation(id, DESCRIPTION_ANNO); + } catch (e) { + return null; + } + }, + + _isLoadInSidebar: function BStore__isLoadInSidebar(id) { + return PlacesUtils.annotations.itemHasAnnotation(id, SIDEBAR_ANNO); + }, + + get _childGUIDsStm() { + return this._getStmt( + "SELECT id AS item_id, guid " + + "FROM moz_bookmarks " + + "WHERE parent = :parent " + + "ORDER BY position"); + }, + _childGUIDsCols: ["item_id", "guid"], + + _getChildGUIDsForId: function _getChildGUIDsForId(itemid) { + let stmt = this._childGUIDsStm; + stmt.params.parent = itemid; + let rows = Async.querySpinningly(stmt, this._childGUIDsCols); + return rows.map(function (row) { + if (row.guid) { + return row.guid; + } + // A GUID hasn't been assigned to this item yet, do this now. + return this.GUIDForId(row.item_id); + }, this); }, // Create a record starting from the weave id (places guid) createRecord: function createRecord(id, collection) { - let item = Async.promiseSpinningly(PlacesSyncUtils.bookmarks.fetch(id)); - if (!item) { // deleted item - let record = new PlacesItem(collection, id); + let placeId = this.idForGUID(id); + let record; + if (placeId <= 0) { // deleted item + record = new PlacesItem(collection, id); record.deleted = true; return record; } - let recordObj = getTypeObject(item.kind); - if (!recordObj) { - this._log.warn("Unknown item type, cannot serialize: " + item.kind); - recordObj = PlacesItem; + let parent = PlacesUtils.bookmarks.getFolderIdForItem(placeId); + switch (PlacesUtils.bookmarks.getItemType(placeId)) { + case PlacesUtils.bookmarks.TYPE_BOOKMARK: + let bmkUri = PlacesUtils.bookmarks.getBookmarkURI(placeId).spec; + if (bmkUri.indexOf("place:") == 0) { + record = new BookmarkQuery(collection, id); + + // Get the actual tag name instead of the local itemId + let folder = bmkUri.match(/[:&]folder=(\d+)/); + try { + // There might not be the tag yet when creating on a new client + if (folder != null) { + folder = folder[1]; + record.folderName = PlacesUtils.bookmarks.getItemTitle(folder); + this._log.trace("query id: " + folder + " = " + record.folderName); + } + } + catch(ex) {} + + // Persist the Smart Bookmark anno, if found. + try { + let anno = PlacesUtils.annotations.getItemAnnotation(placeId, SMART_BOOKMARKS_ANNO); + if (anno != null) { + this._log.trace("query anno: " + SMART_BOOKMARKS_ANNO + + " = " + anno); + record.queryId = anno; + } + } + catch(ex) {} + } + else { + record = new Bookmark(collection, id); + } + record.title = PlacesUtils.bookmarks.getItemTitle(placeId); + + record.parentName = PlacesUtils.bookmarks.getItemTitle(parent); + record.bmkUri = bmkUri; + record.tags = this._getTags(record.bmkUri); + record.keyword = PlacesUtils.bookmarks.getKeywordForBookmark(placeId); + record.description = this._getDescription(placeId); + record.loadInSidebar = this._isLoadInSidebar(placeId); + break; + + case PlacesUtils.bookmarks.TYPE_FOLDER: + if (PlacesUtils.annotations + .itemHasAnnotation(placeId, PlacesUtils.LMANNO_FEEDURI)) { + record = new Livemark(collection, id); + let as = PlacesUtils.annotations; + record.feedUri = as.getItemAnnotation(placeId, PlacesUtils.LMANNO_FEEDURI); + try { + record.siteUri = as.getItemAnnotation(placeId, PlacesUtils.LMANNO_SITEURI); + } catch (ex) {} + } else { + record = new BookmarkFolder(collection, id); + } + + if (parent > 0) + record.parentName = PlacesUtils.bookmarks.getItemTitle(parent); + record.title = PlacesUtils.bookmarks.getItemTitle(placeId); + record.description = this._getDescription(placeId); + record.children = this._getChildGUIDsForId(placeId); + break; + + case PlacesUtils.bookmarks.TYPE_SEPARATOR: + record = new BookmarkSeparator(collection, id); + if (parent > 0) + record.parentName = PlacesUtils.bookmarks.getItemTitle(parent); + // Create a positioning identifier for the separator, used by _mapDupe + record.pos = PlacesUtils.bookmarks.getItemIndex(placeId); + break; + + default: + record = new PlacesItem(collection, id); + this._log.warn("Unknown item type, cannot serialize: " + + PlacesUtils.bookmarks.getItemType(placeId)); } - let record = new recordObj(collection, id); - record.fromSyncBookmark(item); + record.parentid = this.GUIDForId(parent); record.sortindex = this._calculateIndex(record); return record; @@ -997,22 +1118,84 @@ BookmarksStore.prototype = { return this._getStmt( "SELECT frecency " + "FROM moz_places " + - "WHERE url_hash = hash(:url) AND url = :url " + + "WHERE url = :url " + "LIMIT 1"); }, _frecencyCols: ["frecency"], + get _setGUIDStm() { + return this._getStmt( + "UPDATE moz_bookmarks " + + "SET guid = :guid " + + "WHERE id = :item_id"); + }, + + // Some helper functions to handle GUIDs + _setGUID: function _setGUID(id, guid) { + if (!guid) + guid = Utils.makeGUID(); + + let stmt = this._setGUIDStm; + stmt.params.guid = guid; + stmt.params.item_id = id; + Async.querySpinningly(stmt); + return guid; + }, + + get _guidForIdStm() { + return this._getStmt( + "SELECT guid " + + "FROM moz_bookmarks " + + "WHERE id = :item_id"); + }, + _guidForIdCols: ["guid"], + GUIDForId: function GUIDForId(id) { - let guid = Async.promiseSpinningly(PlacesUtils.promiseItemGuid(id)); - return PlacesSyncUtils.bookmarks.guidToSyncId(guid); + let special = kSpecialIds.specialGUIDForId(id); + if (special) + return special; + + let stmt = this._guidForIdStm; + stmt.params.item_id = id; + + // Use the existing GUID if it exists + let result = Async.querySpinningly(stmt, this._guidForIdCols)[0]; + if (result && result.guid) + return result.guid; + + // Give the uri a GUID if it doesn't have one + return this._setGUID(id); }, - idForGUID: function idForGUID(guid) { - // guid might be a String object rather than a string. - guid = PlacesSyncUtils.bookmarks.syncIdToGuid(guid.toString()); + get _idForGUIDStm() { + return this._getStmt( + "SELECT id AS item_id " + + "FROM moz_bookmarks " + + "WHERE guid = :guid"); + }, + _idForGUIDCols: ["item_id"], + + // noCreate is provided as an optional argument to prevent the creation of + // non-existent special records, such as "mobile". + idForGUID: function idForGUID(guid, noCreate) { + if (kSpecialIds.isSpecialGUID(guid)) + return kSpecialIds.specialIdForGUID(guid, !noCreate); - return Async.promiseSpinningly(PlacesUtils.promiseItemId(guid).catch( - ex => -1)); + let stmt = this._idForGUIDStm; + // guid might be a String object rather than a string. + stmt.params.guid = guid.toString(); + + let results = Async.querySpinningly(stmt, this._idForGUIDCols); + this._log.trace("Number of rows matching GUID " + guid + ": " + + results.length); + + // Here's the one we care about: the first. + let result = results[0]; + + if (!result) + return -1; + + return result.item_id; }, _calculateIndex: function _calculateIndex(record) { @@ -1037,48 +1220,107 @@ BookmarksStore.prototype = { return index; }, - getAllIDs: function BStore_getAllIDs() { - let items = {}; - - let query = ` - WITH RECURSIVE - changeRootContents(id) AS ( - VALUES ${getChangeRootIds().map(id => `(${id})`).join(", ")} - UNION ALL - SELECT b.id - FROM moz_bookmarks b - JOIN changeRootContents c ON b.parent = c.id - ) - SELECT guid - FROM changeRootContents - JOIN moz_bookmarks USING (id) - `; - - let statement = this._getStmt(query); - let results = Async.querySpinningly(statement, ["guid"]); - for (let { guid } of results) { - let syncID = PlacesSyncUtils.bookmarks.guidToSyncId(guid); - items[syncID] = { modified: 0, deleted: false }; + _getChildren: function BStore_getChildren(guid, items) { + let node = guid; // the recursion case + if (typeof(node) == "string") { // callers will give us the guid as the first arg + let nodeID = this.idForGUID(guid, true); + if (!nodeID) { + this._log.debug("No node for GUID " + guid + "; returning no children."); + return items; + } + node = this._getNode(nodeID); + } + + if (node.type == node.RESULT_TYPE_FOLDER) { + node.QueryInterface(Ci.nsINavHistoryQueryResultNode); + node.containerOpen = true; + try { + // Remember all the children GUIDs and recursively get more + for (let i = 0; i < node.childCount; i++) { + let child = node.getChild(i); + items[this.GUIDForId(child.itemId)] = true; + this._getChildren(child, items); + } + } + finally { + node.containerOpen = false; + } } return items; }, + /** + * Associates the URI of the item with the provided ID with the + * provided array of tags. + * If the provided ID does not identify an item with a URI, + * returns immediately. + */ + _tagID: function _tagID(itemID, tags) { + if (!itemID || !tags) { + return; + } + + try { + let u = PlacesUtils.bookmarks.getBookmarkURI(itemID); + this._tagURI(u, tags); + } catch (e) { + this._log.warn("Got exception fetching URI for " + itemID + ": not tagging. " + + Utils.exceptionStr(e)); + + // I guess it doesn't have a URI. Don't try to tag it. + return; + } + }, + + /** + * Associate the provided URI with the provided array of tags. + * If the provided URI is falsy, returns immediately. + */ + _tagURI: function _tagURI(bookmarkURI, tags) { + if (!bookmarkURI || !tags) { + return; + } + + // Filter out any null/undefined/empty tags. + tags = tags.filter(t => t); + + // Temporarily tag a dummy URI to preserve tag ids when untagging. + let dummyURI = Utils.makeURI("about:weave#BStore_tagURI"); + PlacesUtils.tagging.tagURI(dummyURI, tags); + PlacesUtils.tagging.untagURI(bookmarkURI, null); + PlacesUtils.tagging.tagURI(bookmarkURI, tags); + PlacesUtils.tagging.untagURI(dummyURI, null); + }, + + getAllIDs: function BStore_getAllIDs() { + let items = {"menu": true, + "toolbar": true}; + for each (let guid in kSpecialIds.guids) { + if (guid != "places" && guid != "tags") + this._getChildren(guid, items); + } + return items; + }, + wipe: function BStore_wipe() { - this.clearPendingDeletions(); - Async.promiseSpinningly(Task.spawn(function* () { + let cb = Async.makeSpinningCallback(); + Task.spawn(function() { // Save a backup before clearing out all bookmarks. yield PlacesBackups.create(null, true); - yield PlacesUtils.bookmarks.eraseEverything({ - source: SOURCE_SYNC, - }); - })); + for each (let guid in kSpecialIds.guids) + if (guid != "places") { + let id = kSpecialIds.specialIdForGUID(guid); + if (id) + PlacesUtils.bookmarks.removeFolderChildren(id); + } + cb(); + }); + cb.wait(); } }; function BookmarksTracker(name, engine) { - this._batchDepth = 0; - this._batchSawScoreIncrement = false; Tracker.call(this, name, engine); Svc.Obs.add("places-shutdown", this); @@ -1086,16 +1328,6 @@ function BookmarksTracker(name, engine) { BookmarksTracker.prototype = { __proto__: Tracker.prototype, - //`_ignore` checks the change source for each observer notification, so we - // don't want to let the engine ignore all changes during a sync. - get ignoreAll() { - return false; - }, - - // Define an empty setter so that the engine doesn't throw a `TypeError` - // setting a read-only property. - set ignoreAll(value) {}, - startTracking: function() { PlacesUtils.bookmarks.addObserver(this, true); Svc.Obs.add("bookmarks-restore-begin", this); @@ -1116,9 +1348,11 @@ BookmarksTracker.prototype = { switch (topic) { case "bookmarks-restore-begin": this._log.debug("Ignoring changes from importing bookmarks."); + this.ignoreAll = true; break; case "bookmarks-restore-success": this._log.debug("Tracking all items on successful import."); + this.ignoreAll = false; this._log.debug("Restore succeeded: wiping server and other clients."); this.engine.service.resetClient([this.name]); @@ -1127,6 +1361,7 @@ BookmarksTracker.prototype = { break; case "bookmarks-restore-failed": this._log.debug("Tracking all items on failed import."); + this.ignoreAll = false; break; } }, @@ -1137,68 +1372,73 @@ BookmarksTracker.prototype = { Ci.nsISupportsWeakReference ]), - addChangedID(id, change) { - if (!id) { - this._log.warn("Attempted to add undefined ID to tracker"); - return false; - } - if (this._ignored.includes(id)) { - return false; - } - let shouldSaveChange = false; - let currentChange = this.changedIDs[id]; - if (currentChange) { - if (typeof currentChange == "number") { - // Allow raw timestamps for backward-compatibility with persisted - // changed IDs. The new format uses tuples to track deleted items. - shouldSaveChange = currentChange < change.modified; - } else { - shouldSaveChange = currentChange.modified < change.modified || - currentChange.deleted != change.deleted; - } - } else { - shouldSaveChange = true; - } - if (shouldSaveChange) { - this._saveChangedID(id, change); - } - return true; - }, - /** * Add a bookmark GUID to be uploaded and bump up the sync score. * - * @param itemId - * The Places item ID of the bookmark to upload. - * @param guid - * The Places GUID of the bookmark to upload. - * @param isTombstone - * Whether we're uploading a tombstone for a removed bookmark. + * @param itemGuid + * GUID of the bookmark to upload. */ - _add: function BMT__add(itemId, guid, isTombstone = false) { - let syncID = PlacesSyncUtils.bookmarks.guidToSyncId(guid); - let info = { modified: Date.now() / 1000, deleted: isTombstone }; - if (this.addChangedID(syncID, info)) { + _add: function BMT__add(itemId, guid) { + guid = kSpecialIds.specialGUIDForId(itemId) || guid; + if (this.addChangedID(guid)) this._upScore(); - } }, - /* Every add/remove/change will trigger a sync for MULTI_DEVICE (except in - a batch operation, where we do it at the end of the batch) */ + /* Every add/remove/change will trigger a sync for MULTI_DEVICE. */ _upScore: function BMT__upScore() { - if (this._batchDepth == 0) { - this.score += SCORE_INCREMENT_XLARGE; - } else { - this._batchSawScoreIncrement = true; + this.score += SCORE_INCREMENT_XLARGE; + }, + + /** + * Determine if a change should be ignored. + * + * @param itemId + * Item under consideration to ignore + * @param folder (optional) + * Folder of the item being changed + */ + _ignore: function BMT__ignore(itemId, folder, guid) { + // Ignore unconditionally if the engine tells us to. + if (this.ignoreAll) + return true; + + // Get the folder id if we weren't given one. + if (folder == null) { + try { + folder = PlacesUtils.bookmarks.getFolderIdForItem(itemId); + } catch (ex) { + this._log.debug("getFolderIdForItem(" + itemId + + ") threw; calling _ensureMobileQuery."); + // I'm guessing that gFIFI can throw, and perhaps that's why + // _ensureMobileQuery is here at all. Try not to call it. + this._ensureMobileQuery(); + folder = PlacesUtils.bookmarks.getFolderIdForItem(itemId); + } + } + + // Ignore changes to tags (folders under the tags folder). + let tags = kSpecialIds.tags; + if (folder == tags) + return true; + + // Ignore tag items (the actual instance of a tag for a bookmark). + if (PlacesUtils.bookmarks.getFolderIdForItem(folder) == tags) + return true; + + // Make sure to remove items that have the exclude annotation. + if (PlacesUtils.annotations.itemHasAnnotation(itemId, EXCLUDEBACKUP_ANNO)) { + this.removeChangedID(guid); + return true; } + + return false; }, onItemAdded: function BMT_onItemAdded(itemId, folder, index, itemType, uri, title, dateAdded, - guid, parentGuid, source) { - if (IGNORED_SOURCES.includes(source)) { + guid, parentGuid) { + if (this._ignore(itemId, folder, guid)) return; - } this._log.trace("onItemAdded: " + itemId); this._add(itemId, guid); @@ -1206,51 +1446,13 @@ BookmarksTracker.prototype = { }, onItemRemoved: function (itemId, parentId, index, type, uri, - guid, parentGuid, source) { - if (IGNORED_SOURCES.includes(source)) { - return; - } - - // Ignore changes to tags (folders under the tags folder). - if (parentId == PlacesUtils.tagsFolderId) { - return; - } - - let grandParentId = -1; - try { - grandParentId = PlacesUtils.bookmarks.getFolderIdForItem(parentId); - } catch (ex) { - // `getFolderIdForItem` can throw if the item no longer exists, such as - // when we've removed a subtree using `removeFolderChildren`. - return; - } - - // Ignore tag items (the actual instance of a tag for a bookmark). - if (grandParentId == PlacesUtils.tagsFolderId) { + guid, parentGuid) { + if (this._ignore(itemId, parentId, guid)) { return; } - /** - * The above checks are incomplete: we can still write tombstones for - * items that we don't track, and upload extraneous roots. - * - * Consider the left pane root: it's a child of the Places root, and has - * children and grandchildren. `PlacesUIUtils` can create, delete, and - * recreate it as needed. We can't determine ancestors when the root or its - * children are deleted, because they've already been removed from the - * database when `onItemRemoved` is called. Likewise, we can't check their - * "exclude from backup" annos, because they've *also* been removed. - * - * So, we end up writing tombstones for the left pane queries and left - * pane root. For good measure, we'll also upload the Places root, because - * it's the parent of the left pane root. - * - * As a workaround, we can track the parent GUID and reconstruct the item's - * ancestry at sync time. This is complicated, and the previous behavior was - * already wrong, so we'll wait for bug 1258127 to fix this generally. - */ this._log.trace("onItemRemoved: " + itemId); - this._add(itemId, guid, /* isTombstone */ true); + this._add(itemId, guid); this._add(parentId, parentGuid); }, @@ -1265,40 +1467,32 @@ BookmarksTracker.prototype = { if (all.length == 0) return; + // Disable handling of notifications while changing the mobile query + this.ignoreAll = true; + let mobile = find(MOBILE_ANNO); - let queryURI = Utils.makeURI("place:folder=" + PlacesUtils.mobileFolderId); - let title = PlacesBundle.GetStringFromName("MobileBookmarksFolderTitle"); + let queryURI = Utils.makeURI("place:folder=" + kSpecialIds.mobile); + let title = Str.sync.get("mobile.label"); // Don't add OR remove the mobile bookmarks if there's nothing. - if (PlacesUtils.bookmarks.getIdForItemAt(PlacesUtils.mobileFolderId, 0) == -1) { + if (PlacesUtils.bookmarks.getIdForItemAt(kSpecialIds.mobile, 0) == -1) { if (mobile.length != 0) - PlacesUtils.bookmarks.removeItem(mobile[0], SOURCE_SYNC); + PlacesUtils.bookmarks.removeItem(mobile[0]); } // Add the mobile bookmarks query if it doesn't exist else if (mobile.length == 0) { - let query = PlacesUtils.bookmarks.insertBookmark(all[0], queryURI, -1, title, /* guid */ null, SOURCE_SYNC); + let query = PlacesUtils.bookmarks.insertBookmark(all[0], queryURI, -1, title); PlacesUtils.annotations.setItemAnnotation(query, ORGANIZERQUERY_ANNO, MOBILE_ANNO, 0, - PlacesUtils.annotations.EXPIRE_NEVER, SOURCE_SYNC); - PlacesUtils.annotations.setItemAnnotation(query, PlacesUtils.EXCLUDE_FROM_BACKUP_ANNO, 1, 0, - PlacesUtils.annotations.EXPIRE_NEVER, SOURCE_SYNC); + PlacesUtils.annotations.EXPIRE_NEVER); + PlacesUtils.annotations.setItemAnnotation(query, EXCLUDEBACKUP_ANNO, 1, 0, + PlacesUtils.annotations.EXPIRE_NEVER); } - // Make sure the existing query URL and title are correct - else { - if (!PlacesUtils.bookmarks.getBookmarkURI(mobile[0]).equals(queryURI)) { - PlacesUtils.bookmarks.changeBookmarkURI(mobile[0], queryURI, - SOURCE_SYNC); - } - let queryTitle = PlacesUtils.bookmarks.getItemTitle(mobile[0]); - if (queryTitle != title) { - PlacesUtils.bookmarks.setItemTitle(mobile[0], title, SOURCE_SYNC); - } - let rootTitle = - PlacesUtils.bookmarks.getItemTitle(PlacesUtils.mobileFolderId); - if (rootTitle != title) { - PlacesUtils.bookmarks.setItemTitle(PlacesUtils.mobileFolderId, title, - SOURCE_SYNC); - } + // Make sure the existing title is correct + else if (PlacesUtils.bookmarks.getItemTitle(mobile[0]) != title) { + PlacesUtils.bookmarks.setItemTitle(mobile[0], title); } + + this.ignoreAll = false; }, // This method is oddly structured, but the idea is to return as quickly as @@ -1306,11 +1500,10 @@ BookmarksTracker.prototype = { // *each change*. onItemChanged: function BMT_onItemChanged(itemId, property, isAnno, value, lastModified, itemType, parentId, - guid, parentGuid, oldValue, - source) { - if (IGNORED_SOURCES.includes(source)) { + guid, parentGuid) { + // Quicker checks first. + if (this.ignoreAll) return; - } if (isAnno && (ANNOS_TO_TRACK.indexOf(property) == -1)) // Ignore annotations except for the ones that we sync. @@ -1320,6 +1513,9 @@ BookmarksTracker.prototype = { if (property == "favicon") return; + if (this._ignore(itemId, parentId, guid)) + return; + this._log.trace("onItemChanged: " + itemId + (", " + property + (isAnno? " (anno)" : "")) + (value ? (" = \"" + value + "\"") : "")); @@ -1328,11 +1524,9 @@ BookmarksTracker.prototype = { onItemMoved: function BMT_onItemMoved(itemId, oldParent, oldIndex, newParent, newIndex, itemType, - guid, oldParentGuid, newParentGuid, - source) { - if (IGNORED_SOURCES.includes(source)) { + guid, oldParentGuid, newParentGuid) { + if (this._ignore(itemId, newParent, guid)) return; - } this._log.trace("onItemMoved: " + itemId); this._add(oldParent, oldParentGuid); @@ -1342,37 +1536,10 @@ BookmarksTracker.prototype = { } // Remove any position annotations now that the user moved the item - PlacesUtils.annotations.removeItemAnnotation(itemId, - PlacesSyncUtils.bookmarks.SYNC_PARENT_ANNO, SOURCE_SYNC); + PlacesUtils.annotations.removeItemAnnotation(itemId, PARENT_ANNO); }, - onBeginUpdateBatch: function () { - ++this._batchDepth; - }, - onEndUpdateBatch: function () { - if (--this._batchDepth === 0 && this._batchSawScoreIncrement) { - this.score += SCORE_INCREMENT_XLARGE; - this._batchSawScoreIncrement = false; - } - }, + onBeginUpdateBatch: function () {}, + onEndUpdateBatch: function () {}, onItemVisited: function () {} }; - -// Returns an array of root IDs to recursively query for synced bookmarks. -// Items in other roots, including tags and organizer queries, will be -// ignored. -function getChangeRootIds() { - return [ - PlacesUtils.bookmarksMenuFolderId, - PlacesUtils.toolbarFolderId, - PlacesUtils.unfiledBookmarksFolderId, - PlacesUtils.mobileFolderId, - ]; -} - -class BookmarksChangeset extends Changeset { - getModifiedTimestamp(id) { - let change = this.changes[id]; - return change ? change.modified : Number.NaN; - } -} diff --git a/services/sync/modules/engines/clients.js b/services/sync/modules/engines/clients.js index 3dd679570..f423242c9 100644 --- a/services/sync/modules/engines/clients.js +++ b/services/sync/modules/engines/clients.js @@ -2,57 +2,24 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -/** - * How does the clients engine work? - * - * - We use 2 files - commands.json and commands-syncing.json. - * - * - At sync upload time, we attempt a rename of commands.json to - * commands-syncing.json, and ignore errors (helps for crash during sync!). - * - We load commands-syncing.json and stash the contents in - * _currentlySyncingCommands which lives for the duration of the upload process. - * - We use _currentlySyncingCommands to build the outgoing records - * - Immediately after successful upload, we delete commands-syncing.json from - * disk (and clear _currentlySyncingCommands). We reconcile our local records - * with what we just wrote in the server, and add failed IDs commands - * back in commands.json - * - Any time we need to "save" a command for future syncs, we load - * commands.json, update it, and write it back out. - */ - this.EXPORTED_SYMBOLS = [ "ClientEngine", "ClientsRec" ]; -var {classes: Cc, interfaces: Ci, utils: Cu} = Components; +const {classes: Cc, interfaces: Ci, utils: Cu} = Components; -Cu.import("resource://services-common/async.js"); Cu.import("resource://services-common/stringbundle.js"); Cu.import("resource://services-sync/constants.js"); Cu.import("resource://services-sync/engines.js"); Cu.import("resource://services-sync/record.js"); -Cu.import("resource://services-sync/resource.js"); Cu.import("resource://services-sync/util.js"); -Cu.import("resource://gre/modules/Services.jsm"); - -XPCOMUtils.defineLazyModuleGetter(this, "fxAccounts", - "resource://gre/modules/FxAccounts.jsm"); const CLIENTS_TTL = 1814400; // 21 days const CLIENTS_TTL_REFRESH = 604800; // 7 days -const STALE_CLIENT_REMOTE_AGE = 604800; // 7 days const SUPPORTED_PROTOCOL_VERSIONS = ["1.1", "1.5"]; -function hasDupeCommand(commands, action) { - if (!commands) { - return false; - } - return commands.some(other => other.command == action.command && - Utils.deepEquals(other.args, action.args)); -} - this.ClientsRec = function ClientsRec(collection, id) { CryptoWrapper.call(this, collection, id); } @@ -66,27 +33,23 @@ Utils.deferGetSet(ClientsRec, "cleartext", ["name", "type", "commands", "version", "protocols", - "formfactor", "os", "appPackage", "application", "device", - "fxaDeviceId"]); + "formfactor", "os", "appPackage", "application", "device"]); this.ClientEngine = function ClientEngine(service) { SyncEngine.call(this, "Clients", service); - // Reset the last sync timestamp on every startup so that we fetch all clients - this.resetLastSync(); + // Reset the client on every startup so that we fetch recent clients + this._resetClient(); } ClientEngine.prototype = { __proto__: SyncEngine.prototype, _storeObj: ClientStore, _recordObj: ClientsRec, _trackerObj: ClientsTracker, - allowSkippedRecord: false, // Always sync client data as it controls other sync behavior - get enabled() { - return true; - }, + get enabled() true, get lastRecordUpload() { return Svc.Prefs.get(this.name + ".lastRecordUpload", 0); @@ -95,31 +58,18 @@ ClientEngine.prototype = { Svc.Prefs.set(this.name + ".lastRecordUpload", Math.floor(value)); }, - get remoteClients() { - // return all non-stale clients for external consumption. - return Object.values(this._store._remoteClients).filter(v => !v.stale); - }, - - remoteClientExists(id) { - let client = this._store._remoteClients[id]; - return !!(client && !client.stale); - }, - // Aggregate some stats on the composition of clients on this account get stats() { let stats = { - hasMobile: this.localType == DEVICE_TYPE_MOBILE, + hasMobile: this.localType == "mobile", names: [this.localName], numClients: 1, }; - for (let id in this._store._remoteClients) { - let {name, type, stale} = this._store._remoteClients[id]; - if (!stale) { - stats.hasMobile = stats.hasMobile || type == DEVICE_TYPE_MOBILE; - stats.names.push(name); - stats.numClients++; - } + for each (let {name, type} in this._store._remoteClients) { + stats.hasMobile = stats.hasMobile || type == "mobile"; + stats.names.push(name); + stats.numClients++; } return stats; @@ -135,11 +85,7 @@ ClientEngine.prototype = { counts.set(this.localType, 1); - for (let id in this._store._remoteClients) { - let record = this._store._remoteClients[id]; - if (record.stale) { - continue; // pretend "stale" records don't exist. - } + for each (let record in this._store._remoteClients) { let type = record.type; if (!counts.has(type)) { counts.set(type, 0); @@ -156,9 +102,7 @@ ClientEngine.prototype = { let localID = Svc.Prefs.get("client.GUID", ""); return localID == "" ? this.localID = Utils.makeGUID() : localID; }, - set localID(value) { - Svc.Prefs.set("client.GUID", value); - }, + set localID(value) Svc.Prefs.set("client.GUID", value), get brandName() { let brand = new StringBundle("chrome://branding/locale/brand.properties"); @@ -166,97 +110,23 @@ ClientEngine.prototype = { }, get localName() { - let name = Utils.getDeviceName(); - // If `getDeviceName` returns the default name, set the pref. FxA registers - // the device before syncing, so we don't need to update the registration - // in this case. - Svc.Prefs.set("client.name", name); - return name; - }, - set localName(value) { - Svc.Prefs.set("client.name", value); - // Update the registration in the background. - fxAccounts.updateDeviceRegistration().catch(error => { - this._log.warn("failed to update fxa device registration", error); - }); - }, + let localName = Svc.Prefs.get("client.name", ""); + if (localName != "") + return localName; - get localType() { - return Utils.getDeviceType(); - }, - set localType(value) { - Svc.Prefs.set("client.type", value); + return this.localName = Utils.getDefaultDeviceName(); }, + set localName(value) Svc.Prefs.set("client.name", value), - getClientName(id) { - if (id == this.localID) { - return this.localName; - } - let client = this._store._remoteClients[id]; - return client ? client.name : ""; - }, - - getClientFxaDeviceId(id) { - if (this._store._remoteClients[id]) { - return this._store._remoteClients[id].fxaDeviceId; - } - return null; - }, + get localType() Svc.Prefs.get("client.type", "desktop"), + set localType(value) Svc.Prefs.set("client.type", value), isMobile: function isMobile(id) { if (this._store._remoteClients[id]) - return this._store._remoteClients[id].type == DEVICE_TYPE_MOBILE; + return this._store._remoteClients[id].type == "mobile"; return false; }, - _readCommands() { - let cb = Async.makeSpinningCallback(); - Utils.jsonLoad("commands", this, commands => cb(null, commands)); - return cb.wait() || {}; - }, - - /** - * Low level function, do not use directly (use _addClientCommand instead). - */ - _saveCommands(commands) { - let cb = Async.makeSpinningCallback(); - Utils.jsonSave("commands", this, commands, error => { - if (error) { - this._log.error("Failed to save JSON outgoing commands", error); - } - cb(); - }); - cb.wait(); - }, - - _prepareCommandsForUpload() { - let cb = Async.makeSpinningCallback(); - Utils.jsonMove("commands", "commands-syncing", this).catch(() => {}) // Ignore errors - .then(() => { - Utils.jsonLoad("commands-syncing", this, commands => cb(null, commands)); - }); - return cb.wait() || {}; - }, - - _deleteUploadedCommands() { - delete this._currentlySyncingCommands; - Async.promiseSpinningly( - Utils.jsonRemove("commands-syncing", this).catch(err => { - this._log.error("Failed to delete syncing-commands file", err); - }) - ); - }, - - _addClientCommand(clientId, command) { - const allCommands = this._readCommands(); - const clientCommands = allCommands[clientId] || []; - if (hasDupeCommand(clientCommands, command)) { - return; - } - allCommands[clientId] = clientCommands.concat(command); - this._saveCommands(allCommands); - }, - _syncStartup: function _syncStartup() { // Reupload new client record periodically. if (Date.now() / 1000 - this.lastRecordUpload > CLIENTS_TTL_REFRESH) { @@ -266,157 +136,9 @@ ClientEngine.prototype = { SyncEngine.prototype._syncStartup.call(this); }, - _processIncoming() { - // Fetch all records from the server. - this.lastSync = 0; - this._incomingClients = {}; - try { - SyncEngine.prototype._processIncoming.call(this); - // Since clients are synced unconditionally, any records in the local store - // that don't exist on the server must be for disconnected clients. Remove - // them, so that we don't upload records with commands for clients that will - // never see them. We also do this to filter out stale clients from the - // tabs collection, since showing their list of tabs is confusing. - for (let id in this._store._remoteClients) { - if (!this._incomingClients[id]) { - this._log.info(`Removing local state for deleted client ${id}`); - this._removeRemoteClient(id); - } - } - // Bug 1264498: Mobile clients don't remove themselves from the clients - // collection when the user disconnects Sync, so we mark as stale clients - // with the same name that haven't synced in over a week. - // (Note we can't simply delete them, or we re-apply them next sync - see - // bug 1287687) - delete this._incomingClients[this.localID]; - let names = new Set([this.localName]); - for (let id in this._incomingClients) { - let record = this._store._remoteClients[id]; - if (!names.has(record.name)) { - names.add(record.name); - continue; - } - let remoteAge = AsyncResource.serverTime - this._incomingClients[id]; - if (remoteAge > STALE_CLIENT_REMOTE_AGE) { - this._log.info(`Hiding stale client ${id} with age ${remoteAge}`); - record.stale = true; - } - } - } finally { - this._incomingClients = null; - } - }, - - _uploadOutgoing() { - this._currentlySyncingCommands = this._prepareCommandsForUpload(); - const clientWithPendingCommands = Object.keys(this._currentlySyncingCommands); - for (let clientId of clientWithPendingCommands) { - if (this._store._remoteClients[clientId] || this.localID == clientId) { - this._modified.set(clientId, 0); - } - } - SyncEngine.prototype._uploadOutgoing.call(this); - }, - - _onRecordsWritten(succeeded, failed) { - // Reconcile the status of the local records with what we just wrote on the - // server - for (let id of succeeded) { - const commandChanges = this._currentlySyncingCommands[id]; - if (id == this.localID) { - if (this.localCommands) { - this.localCommands = this.localCommands.filter(command => !hasDupeCommand(commandChanges, command)); - } - } else { - const clientRecord = this._store._remoteClients[id]; - if (!commandChanges || !clientRecord) { - // should be impossible, else we wouldn't have been writing it. - this._log.warn("No command/No record changes for a client we uploaded"); - continue; - } - // fixup the client record, so our copy of _remoteClients matches what we uploaded. - clientRecord.commands = this._store.createRecord(id); - // we could do better and pass the reference to the record we just uploaded, - // but this will do for now - } - } - - // Re-add failed commands - for (let id of failed) { - const commandChanges = this._currentlySyncingCommands[id]; - if (!commandChanges) { - continue; - } - this._addClientCommand(id, commandChanges); - } - - this._deleteUploadedCommands(); - - // Notify other devices that their own client collection changed - const idsToNotify = succeeded.reduce((acc, id) => { - if (id == this.localID) { - return acc; - } - const fxaDeviceId = this.getClientFxaDeviceId(id); - return fxaDeviceId ? acc.concat(fxaDeviceId) : acc; - }, []); - if (idsToNotify.length > 0) { - this._notifyCollectionChanged(idsToNotify); - } - }, - - _notifyCollectionChanged(ids) { - const message = { - version: 1, - command: "sync:collection_changed", - data: { - collections: ["clients"] - } - }; - fxAccounts.notifyDevices(ids, message, NOTIFY_TAB_SENT_TTL_SECS); - }, - - _syncFinish() { - // Record histograms for our device types, and also write them to a pref - // so non-histogram telemetry (eg, UITelemetry) has easy access to them. - for (let [deviceType, count] of this.deviceTypes) { - let hid; - let prefName = this.name + ".devices."; - switch (deviceType) { - case "desktop": - hid = "WEAVE_DEVICE_COUNT_DESKTOP"; - prefName += "desktop"; - break; - case "mobile": - hid = "WEAVE_DEVICE_COUNT_MOBILE"; - prefName += "mobile"; - break; - default: - this._log.warn(`Unexpected deviceType "${deviceType}" recording device telemetry.`); - continue; - } - Services.telemetry.getHistogramById(hid).add(count); - Svc.Prefs.set(prefName, count); - } - SyncEngine.prototype._syncFinish.call(this); - }, - - _reconcile: function _reconcile(item) { - // Every incoming record is reconciled, so we use this to track the - // contents of the collection on the server. - this._incomingClients[item.id] = item.modified; - - if (!this._store.itemExists(item.id)) { - return true; - } - // Clients are synced unconditionally, so we'll always have new records. - // Unfortunately, this will cause the scheduler to use the immediate sync - // interval for the multi-device case, instead of the active interval. We - // work around this by updating the record during reconciliation, and - // returning false to indicate that the record doesn't need to be applied - // later. - this._store.update(item); - return false; + // Always process incoming items because they might have commands + _reconcile: function _reconcile() { + return true; }, // Treat reset the same as wiping for locally cached clients @@ -426,13 +148,7 @@ ClientEngine.prototype = { _wipeClient: function _wipeClient() { SyncEngine.prototype._resetClient.call(this); - delete this.localCommands; this._store.wipe(); - const logRemoveError = err => this._log.warn("Could not delete json file", err); - Async.promiseSpinningly( - Utils.jsonRemove("commands", this).catch(logRemoveError) - .then(Utils.jsonRemove("commands-syncing", this).catch(logRemoveError)) - ); }, removeClientData: function removeClientData() { @@ -471,6 +187,14 @@ ClientEngine.prototype = { }, /** + * Remove any commands for the local client and mark it for upload. + */ + clearCommands: function clearCommands() { + delete this.localCommands; + this._tracker.addChangedID(this.localID); + }, + + /** * Sends a command+args pair to a specific client. * * @param command Command string @@ -484,17 +208,30 @@ ClientEngine.prototype = { if (!client) { throw new Error("Unknown remote client ID: '" + clientId + "'."); } - if (client.stale) { - throw new Error("Stale remote client ID: '" + clientId + "'."); - } + + // notDupe compares two commands and returns if they are not equal. + let notDupe = function(other) { + return other.command != command || !Utils.deepEquals(other.args, args); + }; let action = { command: command, args: args, }; + if (!client.commands) { + client.commands = [action]; + } + // Add the new action if there are no duplicates. + else if (client.commands.every(notDupe)) { + client.commands.push(action); + } + // It must be a dupe. Skip. + else { + return; + } + this._log.trace("Client " + clientId + " got a new action: " + [command, args]); - this._addClientCommand(clientId, action); this._tracker.addChangedID(clientId); }, @@ -505,17 +242,13 @@ ClientEngine.prototype = { */ processIncomingCommands: function processIncomingCommands() { return this._notify("clients:process-commands", "", function() { - if (!this.localCommands) { - return true; - } + let commands = this.localCommands; - const clearedCommands = this._readCommands()[this.localID]; - const commands = this.localCommands.filter(command => !hasDupeCommand(clearedCommands, command)); + // Immediately clear out the commands as we've got them locally. + this.clearCommands(); - let URIsToDisplay = []; // Process each command in order. - for (let rawCommand of commands) { - let {command, args} = rawCommand; + for each (let {command, args} in commands) { this._log.debug("Processing command: " + command + "(" + args + ")"); let engines = [args[0]]; @@ -536,20 +269,12 @@ ClientEngine.prototype = { this.service.logout(); return false; case "displayURI": - let [uri, clientId, title] = args; - URIsToDisplay.push({ uri, clientId, title }); + this._handleDisplayURI.apply(this, args); break; default: this._log.debug("Received an unknown command: " + command); break; } - // Add the command to the "cleared" commands list - this._addClientCommand(this.localID, rawCommand) - } - this._tracker.addChangedID(this.localID); - - if (URIsToDisplay.length) { - this._handleDisplayURIs(URIsToDisplay); } return true; @@ -588,10 +313,8 @@ ClientEngine.prototype = { if (clientId) { this._sendCommandToClient(command, args, clientId); } else { - for (let [id, record] of Object.entries(this._store._remoteClients)) { - if (!record.stale) { - this._sendCommandToClient(command, args, id); - } + for (let id in this._store._remoteClients) { + this._sendCommandToClient(command, args, id); } } }, @@ -622,11 +345,11 @@ ClientEngine.prototype = { }, /** - * Handle a bunch of received 'displayURI' commands. + * Handle a single received 'displayURI' command. * - * Interested parties should observe the "weave:engine:clients:display-uris" - * topic. The callback will receive an array as the subject parameter - * containing objects with the following keys: + * Interested parties should observe the "weave:engine:clients:display-uri" + * topic. The callback will receive an object as the subject parameter with + * the following keys: * * uri URI (string) that is requested for display. * clientId ID of client that sent the command. @@ -634,24 +357,21 @@ ClientEngine.prototype = { * * The 'data' parameter to the callback will not be defined. * - * @param uris - * An array containing URI objects to display - * @param uris[].uri + * @param uri * String URI that was received - * @param uris[].clientId + * @param clientId * ID of client that sent URI - * @param uris[].title + * @param title * String title of page that URI corresponds to. Older clients may not * send this. */ - _handleDisplayURIs: function _handleDisplayURIs(uris) { - Svc.Obs.notify("weave:engine:clients:display-uris", uris); - }, + _handleDisplayURI: function _handleDisplayURI(uri, clientId, title) { + this._log.info("Received a URI for display: " + uri + " (" + title + + ") from " + clientId); - _removeRemoteClient(id) { - delete this._store._remoteClients[id]; - this._tracker.removeChangedID(id); - }, + let subject = {uri: uri, client: clientId, title: title}; + Svc.Obs.notify("weave:engine:clients:display-uri", subject); + } }; function ClientStore(name, engine) { @@ -660,48 +380,29 @@ function ClientStore(name, engine) { ClientStore.prototype = { __proto__: Store.prototype, - _remoteClients: {}, - create(record) { - this.update(record); + this.update(record) }, update: function update(record) { - if (record.id == this.engine.localID) { - // Only grab commands from the server; local name/type always wins + // Only grab commands from the server; local name/type always wins + if (record.id == this.engine.localID) this.engine.localCommands = record.commands; - } else { + else this._remoteClients[record.id] = record.cleartext; - } }, createRecord: function createRecord(id, collection) { let record = new ClientsRec(collection, id); - const commandsChanges = this.engine._currentlySyncingCommands ? - this.engine._currentlySyncingCommands[id] : - []; - // Package the individual components into a record for the local client if (id == this.engine.localID) { - let cb = Async.makeSpinningCallback(); - fxAccounts.getDeviceId().then(id => cb(null, id), cb); - try { - record.fxaDeviceId = cb.wait(); - } catch(error) { - this._log.warn("failed to get fxa device id", error); - } record.name = this.engine.localName; record.type = this.engine.localType; + record.commands = this.engine.localCommands; record.version = Services.appinfo.version; record.protocols = SUPPORTED_PROTOCOL_VERSIONS; - // Substract the commands we recorded that we've already executed - if (commandsChanges && commandsChanges.length && - this.engine.localCommands && this.engine.localCommands.length) { - record.commands = this.engine.localCommands.filter(command => !hasDupeCommand(commandsChanges, command)); - } - // Optional fields. record.os = Services.appinfo.OS; // "Darwin" record.appPackage = Services.appinfo.ID; @@ -712,20 +413,6 @@ ClientStore.prototype = { // record.formfactor = ""; // Bug 1100722 } else { record.cleartext = this._remoteClients[id]; - - // Add the commands we have to send - if (commandsChanges && commandsChanges.length) { - const recordCommands = record.cleartext.commands || []; - const newCommands = commandsChanges.filter(command => !hasDupeCommand(recordCommands, command)); - record.cleartext.commands = recordCommands.concat(newCommands); - } - - if (record.cleartext.stale) { - // It's almost certainly a logic error for us to upload a record we - // consider stale, so make log noise, but still remove the flag. - this._log.error(`Preparing to upload record ${id} that we consider stale`); - delete record.cleartext.stale; - } } return record; @@ -768,7 +455,7 @@ ClientsTracker.prototype = { break; case "weave:engine:stop-tracking": if (this._enabled) { - Svc.Prefs.ignore("client.name", this); + Svc.Prefs.ignore("clients.name", this); this._enabled = false; } break; diff --git a/services/sync/modules/engines/forms.js b/services/sync/modules/engines/forms.js index 43f79d4f7..d26d57176 100644 --- a/services/sync/modules/engines/forms.js +++ b/services/sync/modules/engines/forms.js @@ -2,11 +2,11 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -this.EXPORTED_SYMBOLS = ['FormEngine', 'FormRec', 'FormValidator']; +this.EXPORTED_SYMBOLS = ['FormEngine', 'FormRec']; -var Cc = Components.classes; -var Ci = Components.interfaces; -var Cu = Components.utils; +const Cc = Components.classes; +const Ci = Components.interfaces; +const Cu = Components.utils; Cu.import("resource://gre/modules/XPCOMUtils.jsm"); Cu.import("resource://services-sync/engines.js"); @@ -14,10 +14,9 @@ Cu.import("resource://services-sync/record.js"); Cu.import("resource://services-common/async.js"); Cu.import("resource://services-sync/util.js"); Cu.import("resource://services-sync/constants.js"); -Cu.import("resource://services-sync/collection_validator.js"); Cu.import("resource://gre/modules/Log.jsm"); -const FORMS_TTL = 3 * 365 * 24 * 60 * 60; // Three years in seconds. +const FORMS_TTL = 5184000; // 60 days this.FormRec = function FormRec(collection, id) { CryptoWrapper.call(this, collection, id); @@ -31,30 +30,26 @@ FormRec.prototype = { Utils.deferGetSet(FormRec, "cleartext", ["name", "value"]); -var FormWrapper = { +let FormWrapper = { _log: Log.repository.getLogger("Sync.Engine.Forms"), _getEntryCols: ["fieldname", "value"], _guidCols: ["guid"], - _promiseSearch: function(terms, searchData) { - return new Promise(resolve => { - let results = []; - let callbacks = { - handleResult(result) { - results.push(result); - }, - handleCompletion(reason) { - resolve(results); - } - }; - Svc.FormHistory.search(terms, searchData, callbacks); - }) - }, - // Do a "sync" search by spinning the event loop until it completes. _searchSpinningly: function(terms, searchData) { - return Async.promiseSpinningly(this._promiseSearch(terms, searchData)); + let results = []; + let cb = Async.makeSpinningCallback(); + let callbacks = { + handleResult: function(result) { + results.push(result); + }, + handleCompletion: function(reason) { + cb(null, results); + } + }; + Svc.FormHistory.search(terms, searchData, callbacks); + return cb.wait(); }, _updateSpinningly: function(changes) { @@ -114,9 +109,7 @@ FormEngine.prototype = { syncPriority: 6, - get prefName() { - return "history"; - }, + get prefName() "history", _findDupe: function _findDupe(item) { return FormWrapper.getGUID(item.name, item.value); @@ -232,9 +225,7 @@ FormTracker.prototype = { observe: function (subject, topic, data) { Tracker.prototype.observe.call(this, subject, topic, data); - if (this.ignoreAll) { - return; - } + switch (topic) { case "satchel-storage-changed": if (data == "formhistory-add" || data == "formhistory-remove") { @@ -250,56 +241,3 @@ FormTracker.prototype = { this.score += SCORE_INCREMENT_MEDIUM; }, }; - - -class FormsProblemData extends CollectionProblemData { - getSummary() { - // We don't support syncing deleted form data, so "clientMissing" isn't a problem - return super.getSummary().filter(entry => - entry.name !== "clientMissing"); - } -} - -class FormValidator extends CollectionValidator { - constructor() { - super("forms", "id", ["name", "value"]); - } - - emptyProblemData() { - return new FormsProblemData(); - } - - getClientItems() { - return FormWrapper._promiseSearch(["guid", "fieldname", "value"], {}); - } - - normalizeClientItem(item) { - return { - id: item.guid, - guid: item.guid, - name: item.fieldname, - fieldname: item.fieldname, - value: item.value, - original: item, - }; - } - - normalizeServerItem(item) { - let res = Object.assign({ - guid: item.id, - fieldname: item.name, - original: item, - }, item); - // Missing `name` or `value` causes the getGUID call to throw - if (item.name !== undefined && item.value !== undefined) { - let guid = FormWrapper.getGUID(item.name, item.value); - if (guid) { - res.guid = guid; - res.id = guid; - res.duped = true; - } - } - - return res; - } -}
\ No newline at end of file diff --git a/services/sync/modules/engines/history.js b/services/sync/modules/engines/history.js index 307d484c1..99ecb4506 100644 --- a/services/sync/modules/engines/history.js +++ b/services/sync/modules/engines/history.js @@ -4,10 +4,10 @@ this.EXPORTED_SYMBOLS = ['HistoryEngine', 'HistoryRec']; -var Cc = Components.classes; -var Ci = Components.interfaces; -var Cu = Components.utils; -var Cr = Components.results; +const Cc = Components.classes; +const Ci = Components.interfaces; +const Cu = Components.utils; +const Cr = Components.results; const HISTORY_TTL = 5184000; // 60 days @@ -44,25 +44,6 @@ HistoryEngine.prototype = { applyIncomingBatchSize: HISTORY_STORE_BATCH_SIZE, syncPriority: 7, - - _processIncoming: function (newitems) { - // We want to notify history observers that a batch operation is underway - // so they don't do lots of work for each incoming record. - let observers = PlacesUtils.history.getObservers(); - function notifyHistoryObservers(notification) { - for (let observer of observers) { - try { - observer[notification](); - } catch (ex) { } - } - } - notifyHistoryObservers("onBeginUpdateBatch"); - try { - return SyncEngine.prototype._processIncoming.call(this, newitems); - } finally { - notifyHistoryObservers("onEndUpdateBatch"); - } - }, }; function HistoryStore(name, engine) { @@ -70,8 +51,7 @@ function HistoryStore(name, engine) { // Explicitly nullify our references to our cached services so we don't leak Svc.Obs.add("places-shutdown", function() { - for (let query in this._stmts) { - let stmt = this._stmts; + for each ([query, stmt] in Iterator(this._stmts)) { stmt.finalize(); } this._stmts = {}; @@ -105,7 +85,7 @@ HistoryStore.prototype = { return this._getStmt( "UPDATE moz_places " + "SET guid = :guid " + - "WHERE url_hash = hash(:page_url) AND url = :page_url"); + "WHERE url = :page_url"); }, // Some helper functions to handle GUIDs @@ -127,7 +107,7 @@ HistoryStore.prototype = { return this._getStmt( "SELECT guid " + "FROM moz_places " + - "WHERE url_hash = hash(:page_url) AND url = :page_url"); + "WHERE url = :page_url"); }, _guidCols: ["guid"], @@ -146,12 +126,12 @@ HistoryStore.prototype = { }, get _visitStm() { - return this._getStmt(`/* do not warn (bug 599936) */ - SELECT visit_type type, visit_date date - FROM moz_historyvisits - JOIN moz_places h ON h.id = place_id - WHERE url_hash = hash(:url) AND url = :url - ORDER BY date DESC LIMIT 20`); + return this._getStmt( + "/* do not warn (bug 599936) */ " + + "SELECT visit_type type, visit_date date " + + "FROM moz_historyvisits " + + "WHERE place_id = (SELECT id FROM moz_places WHERE url = :url) " + + "ORDER BY date DESC LIMIT 10"); }, _visitCols: ["date", "type"], @@ -223,10 +203,7 @@ HistoryStore.prototype = { } else { shouldApply = this._recordToPlaceInfo(record); } - } catch (ex) { - if (Async.isShutdownException(ex)) { - throw ex; - } + } catch(ex) { failed.push(record.id); shouldApply = false; } @@ -299,14 +276,14 @@ HistoryStore.prototype = { if (!visit.date || typeof visit.date != "number") { this._log.warn("Encountered record with invalid visit date: " + visit.date); - continue; + throw "Visit has no date!"; } - if (!visit.type || - !Object.values(PlacesUtils.history.TRANSITIONS).includes(visit.type)) { - this._log.warn("Encountered record with invalid visit type: " + - visit.type + "; ignoring."); - continue; + if (!visit.type || !(visit.type >= PlacesUtils.history.TRANSITION_LINK && + visit.type <= PlacesUtils.history.TRANSITION_FRAMED_LINK)) { + this._log.warn("Encountered record with invalid visit type: " + + visit.type); + throw "Invalid visit type!"; } // Dates need to be integers. @@ -317,7 +294,6 @@ HistoryStore.prototype = { // overwritten. continue; } - visit.visitDate = visit.date; visit.transitionType = visit.type; k += 1; @@ -369,9 +345,7 @@ HistoryStore.prototype = { }, wipe: function HistStore_wipe() { - let cb = Async.makeSyncCallback(); - PlacesUtils.history.clear().then(result => {cb(null, result)}, err => {cb(err)}); - return Async.waitForSyncCallback(cb); + PlacesUtils.history.removeAllPages(); } }; diff --git a/services/sync/modules/engines/passwords.js b/services/sync/modules/engines/passwords.js index 51db49a0a..994b59767 100644 --- a/services/sync/modules/engines/passwords.js +++ b/services/sync/modules/engines/passwords.js @@ -2,16 +2,14 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -this.EXPORTED_SYMBOLS = ['PasswordEngine', 'LoginRec', 'PasswordValidator']; +this.EXPORTED_SYMBOLS = ['PasswordEngine', 'LoginRec']; -var {classes: Cc, interfaces: Ci, utils: Cu} = Components; +const {classes: Cc, interfaces: Ci, utils: Cu} = Components; Cu.import("resource://services-sync/record.js"); Cu.import("resource://services-sync/constants.js"); -Cu.import("resource://services-sync/collection_validator.js"); Cu.import("resource://services-sync/engines.js"); Cu.import("resource://services-sync/util.js"); -Cu.import("resource://services-common/async.js"); this.LoginRec = function LoginRec(collection, id) { CryptoWrapper.call(this, collection, id); @@ -24,7 +22,6 @@ LoginRec.prototype = { Utils.deferGetSet(LoginRec, "cleartext", [ "hostname", "formSubmitURL", "httpRealm", "username", "password", "usernameField", "passwordField", - "timeCreated", "timePasswordChanged", ]); @@ -70,10 +67,7 @@ PasswordEngine.prototype = { Svc.Prefs.set("deletePwdFxA", true); Svc.Prefs.reset("deletePwd"); // The old prefname we previously used. } catch (ex) { - if (Async.isShutdownException(ex)) { - throw ex; - } - this._log.debug("Password deletes failed", ex); + this._log.debug("Password deletes failed: " + Utils.exceptionStr(ex)); } } }, @@ -89,7 +83,7 @@ PasswordEngine.prototype = { this._store._sleep(0); // Yield back to main thread after synchronous operation. // Look for existing logins that match the hostname, but ignore the password. - for (let local of logins) { + for each (let local in logins) { if (login.matches(local, true) && local instanceof Ci.nsILoginMetaInfo) { return local.guid; } @@ -104,13 +98,6 @@ function PasswordStore(name, engine) { PasswordStore.prototype = { __proto__: Store.prototype, - _newPropertyBag: function () { - return Cc["@mozilla.org/hash-property-bag;1"].createInstance(Ci.nsIWritablePropertyBag2); - }, - - /** - * Return an instance of nsILoginInfo (and, implicitly, nsILoginMetaInfo). - */ _nsLoginInfoFromRecord: function (record) { function nullUndefined(x) { return (x == undefined) ? null : x; @@ -131,21 +118,13 @@ PasswordStore.prototype = { record.password, record.usernameField, record.passwordField); - info.QueryInterface(Ci.nsILoginMetaInfo); info.guid = record.id; - if (record.timeCreated) { - info.timeCreated = record.timeCreated; - } - if (record.timePasswordChanged) { - info.timePasswordChanged = record.timePasswordChanged; - } - return info; }, _getLoginFromGUID: function (id) { - let prop = this._newPropertyBag(); + let prop = Cc["@mozilla.org/hash-property-bag;1"].createInstance(Ci.nsIWritablePropertyBag2); prop.setPropertyAsAUTF8String("guid", id); let logins = Services.logins.searchLogins({}, prop); @@ -190,7 +169,8 @@ PasswordStore.prototype = { return; } - let prop = this._newPropertyBag(); + let prop = Cc["@mozilla.org/hash-property-bag;1"] + .createInstance(Ci.nsIWritablePropertyBag2); prop.setPropertyAsAUTF8String("guid", newID); Services.logins.modifyLogin(oldLogin, prop); @@ -217,11 +197,6 @@ PasswordStore.prototype = { record.usernameField = login.usernameField; record.passwordField = login.passwordField; - // Optional fields. - login.QueryInterface(Ci.nsILoginMetaInfo); - record.timeCreated = login.timeCreated; - record.timePasswordChanged = login.timePasswordChanged; - return record; }, @@ -237,7 +212,8 @@ PasswordStore.prototype = { try { Services.logins.addLogin(login); } catch(ex) { - this._log.debug(`Adding record ${record.id} resulted in exception`, ex); + this._log.debug("Adding record " + record.id + + " resulted in exception " + Utils.exceptionStr(ex)); } }, @@ -269,7 +245,9 @@ PasswordStore.prototype = { try { Services.logins.modifyLogin(loginItem, newinfo); } catch(ex) { - this._log.debug(`Modifying record ${record.id} resulted in exception; not modifying`, ex); + this._log.debug("Modifying record " + record.id + + " resulted in exception " + Utils.exceptionStr(ex) + + ". Not modifying."); } }, @@ -326,46 +304,3 @@ PasswordTracker.prototype = { } }, }; - -class PasswordValidator extends CollectionValidator { - constructor() { - super("passwords", "id", [ - "hostname", - "formSubmitURL", - "httpRealm", - "password", - "passwordField", - "username", - "usernameField", - ]); - } - - getClientItems() { - let logins = Services.logins.getAllLogins({}); - let syncHosts = Utils.getSyncCredentialsHosts() - let result = logins.map(l => l.QueryInterface(Ci.nsILoginMetaInfo)) - .filter(l => !syncHosts.has(l.hostname)); - return Promise.resolve(result); - } - - normalizeClientItem(item) { - return { - id: item.guid, - guid: item.guid, - hostname: item.hostname, - formSubmitURL: item.formSubmitURL, - httpRealm: item.httpRealm, - password: item.password, - passwordField: item.passwordField, - username: item.username, - usernameField: item.usernameField, - original: item, - } - } - - normalizeServerItem(item) { - return Object.assign({ guid: item.id }, item); - } -} - - diff --git a/services/sync/modules/engines/prefs.js b/services/sync/modules/engines/prefs.js index 9ceeb9ac6..82091d5b4 100644 --- a/services/sync/modules/engines/prefs.js +++ b/services/sync/modules/engines/prefs.js @@ -4,11 +4,11 @@ this.EXPORTED_SYMBOLS = ['PrefsEngine', 'PrefRec']; -var Cc = Components.classes; -var Ci = Components.interfaces; -var Cu = Components.utils; +const Cc = Components.classes; +const Ci = Components.interfaces; +const Cu = Components.utils; -const PREF_SYNC_PREFS_PREFIX = "services.sync.prefs.sync."; +const SYNC_PREFS_PREFIX = "services.sync.prefs.sync."; Cu.import("resource://services-sync/engines.js"); Cu.import("resource://services-sync/record.js"); @@ -42,7 +42,6 @@ PrefsEngine.prototype = { version: 2, syncPriority: 1, - allowSkippedRecord: false, getChangedIDs: function () { // No need for a proper timestamp (no conflict resolution needed). @@ -88,45 +87,37 @@ PrefStore.prototype = { _getSyncPrefs: function () { let syncPrefs = Cc["@mozilla.org/preferences-service;1"] .getService(Ci.nsIPrefService) - .getBranch(PREF_SYNC_PREFS_PREFIX) + .getBranch(SYNC_PREFS_PREFIX) .getChildList("", {}); // Also sync preferences that determine which prefs get synced. - let controlPrefs = syncPrefs.map(pref => PREF_SYNC_PREFS_PREFIX + pref); + let controlPrefs = syncPrefs.map(pref => SYNC_PREFS_PREFIX + pref); return controlPrefs.concat(syncPrefs); }, _isSynced: function (pref) { - return pref.startsWith(PREF_SYNC_PREFS_PREFIX) || - this._prefs.get(PREF_SYNC_PREFS_PREFIX + pref, false); + return pref.startsWith(SYNC_PREFS_PREFIX) || + this._prefs.get(SYNC_PREFS_PREFIX + pref, false); }, _getAllPrefs: function () { let values = {}; - for (let pref of this._getSyncPrefs()) { + for each (let pref in this._getSyncPrefs()) { if (this._isSynced(pref)) { - // Missing and default prefs get the null value. - values[pref] = this._prefs.isSet(pref) ? this._prefs.get(pref, null) : null; + // Missing prefs get the null value. + values[pref] = this._prefs.get(pref, null); } } return values; }, - _updateLightWeightTheme (themeID) { - let themeObject = null; - if (themeID) { - themeObject = LightweightThemeManager.getUsedTheme(themeID); - } - LightweightThemeManager.currentTheme = themeObject; - }, - _setAllPrefs: function (values) { - let selectedThemeIDPref = "lightweightThemes.selectedThemeID"; - let selectedThemeIDBefore = this._prefs.get(selectedThemeIDPref, null); - let selectedThemeIDAfter = selectedThemeIDBefore; + let enabledPref = "lightweightThemes.isThemeSelected"; + let enabledBefore = this._prefs.get(enabledPref, false); + let prevTheme = LightweightThemeManager.currentTheme; // Update 'services.sync.prefs.sync.foo.pref' before 'foo.pref', otherwise // _isSynced returns false when 'foo.pref' doesn't exist (e.g., on a new device). - let prefs = Object.keys(values).sort(a => -a.indexOf(PREF_SYNC_PREFS_PREFIX)); + let prefs = Object.keys(values).sort(a => -a.indexOf(SYNC_PREFS_PREFIX)); for (let pref of prefs) { if (!this._isSynced(pref)) { continue; @@ -134,30 +125,26 @@ PrefStore.prototype = { let value = values[pref]; - switch (pref) { - // Some special prefs we don't want to set directly. - case selectedThemeIDPref: - selectedThemeIDAfter = value; - break; - - // default is to just set the pref - default: - if (value == null) { - // Pref has gone missing. The best we can do is reset it. - this._prefs.reset(pref); - } else { - try { - this._prefs.set(pref, value); - } catch(ex) { - this._log.trace("Failed to set pref: " + pref + ": " + ex); - } - } + // Pref has gone missing. The best we can do is reset it. + if (value == null) { + this._prefs.reset(pref); + continue; } + + try { + this._prefs.set(pref, value); + } catch(ex) { + this._log.trace("Failed to set pref: " + pref + ": " + ex); + } } - // Notify the lightweight theme manager if the selected theme has changed. - if (selectedThemeIDBefore != selectedThemeIDAfter) { - this._updateLightWeightTheme(selectedThemeIDAfter); + // Notify the lightweight theme manager of all the new values + let enabledNow = this._prefs.get(enabledPref, false); + if (enabledBefore && !enabledNow) { + LightweightThemeManager.currentTheme = null; + } else if (enabledNow && LightweightThemeManager.usedThemes[0] != prevTheme) { + LightweightThemeManager.currentTheme = null; + LightweightThemeManager.currentTheme = LightweightThemeManager.usedThemes[0]; } }, @@ -261,8 +248,8 @@ PrefTracker.prototype = { case "nsPref:changed": // Trigger a sync for MULTI-DEVICE for a change that determines // which prefs are synced or a regular pref change. - if (data.indexOf(PREF_SYNC_PREFS_PREFIX) == 0 || - this._prefs.get(PREF_SYNC_PREFS_PREFIX + data, false)) { + if (data.indexOf(SYNC_PREFS_PREFIX) == 0 || + this._prefs.get(SYNC_PREFS_PREFIX + data, false)) { this.score += SCORE_INCREMENT_XLARGE; this.modified = true; this._log.trace("Preference " + data + " changed"); diff --git a/services/sync/modules/engines/tabs.js b/services/sync/modules/engines/tabs.js index 45ece4a23..1fce737d2 100644 --- a/services/sync/modules/engines/tabs.js +++ b/services/sync/modules/engines/tabs.js @@ -4,7 +4,7 @@ this.EXPORTED_SYMBOLS = ["TabEngine", "TabSetRecord"]; -var {classes: Cc, interfaces: Ci, utils: Cu} = Components; +const {classes: Cc, interfaces: Ci, utils: Cu} = Components; const TABS_TTL = 604800; // 7 days. const TAB_ENTRIES_LIMIT = 25; // How many URLs to include in tab history. @@ -43,11 +43,6 @@ TabEngine.prototype = { _storeObj: TabStore, _trackerObj: TabTracker, _recordObj: TabSetRecord, - // A flag to indicate if we have synced in this session. This is to help - // consumers of remote tabs that may want to differentiate between "I've an - // empty tab list as I haven't yet synced" vs "I've an empty tab list - // as there really are no tabs" - hasSyncedThisSession: false, syncPriority: 3, @@ -72,7 +67,6 @@ TabEngine.prototype = { SyncEngine.prototype._resetClient.call(this); this._store.wipe(); this._tracker.modified = true; - this.hasSyncedThisSession = false; }, removeClientData: function () { @@ -100,12 +94,7 @@ TabEngine.prototype = { } return SyncEngine.prototype._reconcile.call(this, item); - }, - - _syncFinish() { - this.hasSyncedThisSession = true; - return SyncEngine.prototype._syncFinish.call(this); - }, + } }; @@ -145,7 +134,7 @@ TabStore.prototype = { } for (let tab of win.gBrowser.tabs) { - let tabState = this.getTabState(tab); + tabState = this.getTabState(tab); // Make sure there are history entries to look at. if (!tabState || !tabState.entries.length) { @@ -165,11 +154,6 @@ TabStore.prototype = { continue; } - if (current.url.length >= (MAX_UPLOAD_BYTES - 1000)) { - this._log.trace("Skipping over-long URL."); - continue; - } - // The element at `index` is the current page. Previous URLs were // previously visited URLs; subsequent URLs are in the 'forward' stack, // which we can't represent in Sync, so we truncate here. @@ -189,9 +173,7 @@ TabStore.prototype = { allTabs.push({ title: current.title || "", urlHistory: urls, - icon: tabState.image || - (tabState.attributes && tabState.attributes.image) || - "", + icon: tabState.attributes && tabState.attributes.image || "", lastUsed: Math.floor((tabState.lastAccessed || 0) / 1000), }); } @@ -265,9 +247,27 @@ TabStore.prototype = { create: function (record) { this._log.debug("Adding remote tabs from " + record.clientName); - this._remoteClients[record.id] = Object.assign({}, record.cleartext, { - lastModified: record.modified - }); + this._remoteClients[record.id] = record.cleartext; + + // Lose some precision, but that's good enough (seconds). + let roundModify = Math.floor(record.modified / 1000); + let notifyState = Svc.Prefs.get("notifyTabState"); + + // If there's no existing pref, save this first modified time. + if (notifyState == null) { + Svc.Prefs.set("notifyTabState", roundModify); + return; + } + + // Don't change notifyState if it's already 0 (don't notify). + if (notifyState == 0) { + return; + } + + // We must have gotten a new tab that isn't the same as last time. + if (notifyState != roundModify) { + Svc.Prefs.set("notifyTabState", 0); + } }, update: function (record) { @@ -302,14 +302,10 @@ TabTracker.prototype = { _registerListenersForWindow: function (window) { this._log.trace("Registering tab listeners in window"); - for (let topic of this._topics) { + for each (let topic in this._topics) { window.addEventListener(topic, this.onTab, false); } window.addEventListener("unload", this._unregisterListeners, false); - // If it's got a tab browser we can listen for things like navigation. - if (window.gBrowser) { - window.gBrowser.addProgressListener(this); - } }, _unregisterListeners: function (event) { @@ -319,12 +315,9 @@ TabTracker.prototype = { _unregisterListenersForWindow: function (window) { this._log.trace("Removing tab listeners in window"); window.removeEventListener("unload", this._unregisterListeners, false); - for (let topic of this._topics) { + for each (let topic in this._topics) { window.removeEventListener(topic, this.onTab, false); } - if (window.gBrowser) { - window.gBrowser.removeProgressListener(this); - } }, startTracking: function () { @@ -380,14 +373,4 @@ TabTracker.prototype = { this.score += SCORE_INCREMENT_SMALL; } }, - - // web progress listeners. - onLocationChange: function (webProgress, request, location, flags) { - // We only care about top-level location changes which are not in the same - // document. - if (webProgress.isTopLevel && - ((flags & Ci.nsIWebProgressListener.LOCATION_CHANGE_SAME_DOCUMENT) == 0)) { - this.modified = true; - } - }, }; diff --git a/services/sync/modules/healthreport.jsm b/services/sync/modules/healthreport.jsm new file mode 100644 index 000000000..47161c095 --- /dev/null +++ b/services/sync/modules/healthreport.jsm @@ -0,0 +1,262 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +this.EXPORTED_SYMBOLS = [ + "SyncProvider", +]; + +const {classes: Cc, interfaces: Ci, utils: Cu} = Components; + +Cu.import("resource://gre/modules/Metrics.jsm", this); +Cu.import("resource://gre/modules/Promise.jsm", this); +Cu.import("resource://gre/modules/Services.jsm", this); +Cu.import("resource://gre/modules/XPCOMUtils.jsm", this); + +const DAILY_LAST_NUMERIC_FIELD = {type: Metrics.Storage.FIELD_DAILY_LAST_NUMERIC}; +const DAILY_LAST_TEXT_FIELD = {type: Metrics.Storage.FIELD_DAILY_LAST_TEXT}; +const DAILY_COUNTER_FIELD = {type: Metrics.Storage.FIELD_DAILY_COUNTER}; + +XPCOMUtils.defineLazyModuleGetter(this, "Weave", + "resource://services-sync/main.js"); + +function SyncMeasurement1() { + Metrics.Measurement.call(this); +} + +SyncMeasurement1.prototype = Object.freeze({ + __proto__: Metrics.Measurement.prototype, + + name: "sync", + version: 1, + + fields: { + enabled: DAILY_LAST_NUMERIC_FIELD, + preferredProtocol: DAILY_LAST_TEXT_FIELD, + activeProtocol: DAILY_LAST_TEXT_FIELD, + syncStart: DAILY_COUNTER_FIELD, + syncSuccess: DAILY_COUNTER_FIELD, + syncError: DAILY_COUNTER_FIELD, + }, +}); + +function SyncDevicesMeasurement1() { + Metrics.Measurement.call(this); +} + +SyncDevicesMeasurement1.prototype = Object.freeze({ + __proto__: Metrics.Measurement.prototype, + + name: "devices", + version: 1, + + fields: {}, + + shouldIncludeField: function (name) { + return true; + }, + + fieldType: function (name) { + return Metrics.Storage.FIELD_DAILY_COUNTER; + }, +}); + +function SyncMigrationMeasurement1() { + Metrics.Measurement.call(this); +} + +SyncMigrationMeasurement1.prototype = Object.freeze({ + __proto__: Metrics.Measurement.prototype, + + name: "migration", + version: 1, + + fields: { + state: DAILY_LAST_TEXT_FIELD, // last "user" or "internal" state we saw for the day + accepted: DAILY_COUNTER_FIELD, // number of times user tried to start migration + declined: DAILY_COUNTER_FIELD, // number of times user closed nagging infobar + unlinked: DAILY_LAST_NUMERIC_FIELD, // did the user decline and unlink + }, +}); + +this.SyncProvider = function () { + Metrics.Provider.call(this); +}; +SyncProvider.prototype = Object.freeze({ + __proto__: Metrics.Provider.prototype, + + name: "org.mozilla.sync", + + measurementTypes: [ + SyncDevicesMeasurement1, + SyncMeasurement1, + SyncMigrationMeasurement1, + ], + + _OBSERVERS: [ + "weave:service:sync:start", + "weave:service:sync:finish", + "weave:service:sync:error", + "fxa-migration:state-changed", + "fxa-migration:internal-state-changed", + "fxa-migration:internal-telemetry", + ], + + postInit: function () { + for (let o of this._OBSERVERS) { + Services.obs.addObserver(this, o, false); + } + + return Promise.resolve(); + }, + + onShutdown: function () { + for (let o of this._OBSERVERS) { + Services.obs.removeObserver(this, o); + } + + return Promise.resolve(); + }, + + observe: function (subject, topic, data) { + switch (topic) { + case "weave:service:sync:start": + case "weave:service:sync:finish": + case "weave:service:sync:error": + return this._observeSync(subject, topic, data); + + case "fxa-migration:state-changed": + case "fxa-migration:internal-state-changed": + case "fxa-migration:internal-telemetry": + return this._observeMigration(subject, topic, data); + } + Cu.reportError("unexpected topic in sync healthreport provider: " + topic); + }, + + _observeSync: function (subject, topic, data) { + let field; + switch (topic) { + case "weave:service:sync:start": + field = "syncStart"; + break; + + case "weave:service:sync:finish": + field = "syncSuccess"; + break; + + case "weave:service:sync:error": + field = "syncError"; + break; + + default: + Cu.reportError("unexpected sync topic in sync healthreport provider: " + topic); + return; + } + + let m = this.getMeasurement(SyncMeasurement1.prototype.name, + SyncMeasurement1.prototype.version); + return this.enqueueStorageOperation(function recordSyncEvent() { + return m.incrementDailyCounter(field); + }); + }, + + _observeMigration: function(subject, topic, data) { + switch (topic) { + case "fxa-migration:state-changed": + case "fxa-migration:internal-state-changed": { + // We record both "user" and "internal" states in the same field. This + // works for us as user state is always null when there is an internal + // state. + if (!data) { + return; // we don't count the |null| state + } + let m = this.getMeasurement(SyncMigrationMeasurement1.prototype.name, + SyncMigrationMeasurement1.prototype.version); + return this.enqueueStorageOperation(function() { + return m.setDailyLastText("state", data); + }); + } + + case "fxa-migration:internal-telemetry": { + // |data| is our field name. + let m = this.getMeasurement(SyncMigrationMeasurement1.prototype.name, + SyncMigrationMeasurement1.prototype.version); + return this.enqueueStorageOperation(function() { + switch (data) { + case "accepted": + case "declined": + return m.incrementDailyCounter(data); + case "unlinked": + return m.setDailyLastNumeric(data, 1); + default: + Cu.reportError("Unexpected migration field in sync healthreport provider: " + data); + return Promise.resolve(); + } + }); + } + + default: + Cu.reportError("unexpected migration topic in sync healthreport provider: " + topic); + return; + } + }, + + collectDailyData: function () { + return this.storage.enqueueTransaction(this._populateDailyData.bind(this)); + }, + + _populateDailyData: function* () { + let m = this.getMeasurement(SyncMeasurement1.prototype.name, + SyncMeasurement1.prototype.version); + + let svc = Cc["@mozilla.org/weave/service;1"] + .getService(Ci.nsISupports) + .wrappedJSObject; + + let enabled = svc.enabled; + yield m.setDailyLastNumeric("enabled", enabled ? 1 : 0); + + // preferredProtocol is constant and only changes as the client + // evolves. + yield m.setDailyLastText("preferredProtocol", "1.5"); + + let protocol = svc.fxAccountsEnabled ? "1.5" : "1.1"; + yield m.setDailyLastText("activeProtocol", protocol); + + if (!enabled) { + return; + } + + // Before grabbing more information, be sure the Sync service + // is fully initialized. This has the potential to initialize + // Sync on the spot. This may be undesired if Sync appears to + // be enabled but it really isn't. That responsibility should + // be up to svc.enabled to not return false positives, however. + yield svc.whenLoaded(); + + if (Weave.Status.service != Weave.STATUS_OK) { + return; + } + + // Device types are dynamic. So we need to dynamically create fields if + // they don't exist. + let dm = this.getMeasurement(SyncDevicesMeasurement1.prototype.name, + SyncDevicesMeasurement1.prototype.version); + let devices = Weave.Service.clientsEngine.deviceTypes; + for (let [field, count] of devices) { + let hasField = this.storage.hasFieldFromMeasurement(dm.id, field, + this.storage.FIELD_DAILY_LAST_NUMERIC); + let fieldID; + if (hasField) { + fieldID = this.storage.fieldIDFromMeasurement(dm.id, field); + } else { + fieldID = yield this.storage.registerField(dm.id, field, + this.storage.FIELD_DAILY_LAST_NUMERIC); + } + + yield this.storage.setDailyLastNumericFromFieldID(fieldID, count); + } + }, +}); diff --git a/services/sync/modules/identity.js b/services/sync/modules/identity.js index b4da8c0bb..2bee13b5b 100644 --- a/services/sync/modules/identity.js +++ b/services/sync/modules/identity.js @@ -6,14 +6,13 @@ this.EXPORTED_SYMBOLS = ["IdentityManager"]; -var {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components; +const {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components; Cu.import("resource://gre/modules/XPCOMUtils.jsm"); Cu.import("resource://gre/modules/Promise.jsm"); Cu.import("resource://services-sync/constants.js"); Cu.import("resource://gre/modules/Log.jsm"); Cu.import("resource://services-sync/util.js"); -Cu.import("resource://services-common/async.js"); // Lazy import to prevent unnecessary load on startup. for (let symbol of ["BulkKeyBundle", "SyncKeyBundle"]) { @@ -85,14 +84,18 @@ IdentityManager.prototype = { _syncKeyBundle: null, /** - * Initialize the identity provider. + * Initialize the identity provider. Returns a promise that is resolved + * when initialization is complete and the provider can be queried for + * its state */ initialize: function() { // Nothing to do for this identity provider. + return Promise.resolve(); }, finalize: function() { // Nothing to do for this identity provider. + return Promise.resolve(); }, /** @@ -111,6 +114,14 @@ IdentityManager.prototype = { return Promise.resolve(); }, + /** + * Indicates if the identity manager is still initializing + */ + get readyToAuthenticate() { + // We initialize in a fully sync manner, so we are always finished. + return true; + }, + get account() { return Svc.Prefs.get("account", this.username); }, @@ -195,7 +206,7 @@ IdentityManager.prototype = { return null; } - for (let login of this._getLogins(PWDMGR_PASSWORD_REALM)) { + for each (let login in this._getLogins(PWDMGR_PASSWORD_REALM)) { if (login.username.toLowerCase() == username) { // It should already be UTF-8 encoded, but we don't take any chances. this._basicPassword = Utils.encodeUTF8(login.password); @@ -249,7 +260,7 @@ IdentityManager.prototype = { return null; } - for (let login of this._getLogins(PWDMGR_PASSPHRASE_REALM)) { + for each (let login in this._getLogins(PWDMGR_PASSPHRASE_REALM)) { if (login.username.toLowerCase() == username) { this._syncKey = login.password; } @@ -326,7 +337,7 @@ IdentityManager.prototype = { try { this._syncKeyBundle = new SyncKeyBundle(this.username, this.syncKey); } catch (ex) { - this._log.warn("Failed to create sync bundle", ex); + this._log.warn(Utils.exceptionStr(ex)); return null; } } @@ -400,7 +411,7 @@ IdentityManager.prototype = { this._setLogin(PWDMGR_PASSWORD_REALM, this.username, this._basicPassword); } else { - for (let login of this._getLogins(PWDMGR_PASSWORD_REALM)) { + for each (let login in this._getLogins(PWDMGR_PASSWORD_REALM)) { Services.logins.removeLogin(login); } } @@ -412,7 +423,7 @@ IdentityManager.prototype = { if (this._syncKey) { this._setLogin(PWDMGR_PASSPHRASE_REALM, this.username, this._syncKey); } else { - for (let login of this._getLogins(PWDMGR_PASSPHRASE_REALM)) { + for each (let login in this._getLogins(PWDMGR_PASSPHRASE_REALM)) { Services.logins.removeLogin(login); } } @@ -447,9 +458,6 @@ IdentityManager.prototype = { try { service.recordManager.get(service.storageURL + "meta/fxa_credentials"); } catch (ex) { - if (Async.isShutdownException(ex)) { - throw ex; - } this._log.warn("Failed to pre-fetch the migration sentinel", ex); } }, @@ -469,7 +477,7 @@ IdentityManager.prototype = { */ _setLogin: function _setLogin(realm, username, password) { let exists = false; - for (let login of this._getLogins(realm)) { + for each (let login in this._getLogins(realm)) { if (login.username == username && login.password == password) { exists = true; } else { @@ -505,7 +513,7 @@ IdentityManager.prototype = { deleteSyncCredentials: function deleteSyncCredentials() { for (let host of this._getSyncCredentialsHosts()) { let logins = Services.logins.findLogins({}, host, "", ""); - for (let login of logins) { + for each (let login in logins) { Services.logins.removeLogin(login); } } @@ -593,13 +601,4 @@ IdentityManager.prototype = { // Do nothing for Sync 1.1. return {accepted: true}; }, - - // Tell Sync what the login status should be if it saw a 401 fetching - // info/collections as part of login verification (typically immediately - // after login.) - // In our case it means an authoritative "password is incorrect". - loginStatusFromVerification404() { - return LOGIN_FAILED_LOGIN_REJECTED; - } - }; diff --git a/services/sync/modules/jpakeclient.js b/services/sync/modules/jpakeclient.js index 625dc91b6..10f405371 100644 --- a/services/sync/modules/jpakeclient.js +++ b/services/sync/modules/jpakeclient.js @@ -4,7 +4,7 @@ this.EXPORTED_SYMBOLS = ["JPAKEClient", "SendCredentialsController"]; -var {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components; +const {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components; Cu.import("resource://gre/modules/Log.jsm"); Cu.import("resource://services-common/rest.js"); @@ -281,7 +281,8 @@ JPAKEClient.prototype = { let rng = Cc["@mozilla.org/security/random-generator;1"] .createInstance(Ci.nsIRandomGenerator); let bytes = rng.generateRandomBytes(JPAKE_LENGTH_CLIENTID / 2); - this._clientID = bytes.map(byte => ("0" + byte.toString(16)).slice(-2)).join(""); + this._clientID = [("0" + byte.toString(16)).slice(-2) + for each (byte in bytes)].join(""); }, _createSecret: function _createSecret() { @@ -290,7 +291,8 @@ JPAKEClient.prototype = { let rng = Cc["@mozilla.org/security/random-generator;1"] .createInstance(Ci.nsIRandomGenerator); let bytes = rng.generateRandomBytes(JPAKE_LENGTH_SECRET); - return bytes.map(byte => key[Math.floor(byte * key.length / 256)]).join(""); + return [key[Math.floor(byte * key.length / 256)] + for each (byte in bytes)].join(""); }, _newRequest: function _newRequest(uri) { diff --git a/services/sync/modules/keys.js b/services/sync/modules/keys.js index b93de7f31..bf909bdc2 100644 --- a/services/sync/modules/keys.js +++ b/services/sync/modules/keys.js @@ -9,7 +9,7 @@ this.EXPORTED_SYMBOLS = [ "SyncKeyBundle" ]; -var {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components; +const {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components; Cu.import("resource://services-sync/constants.js"); Cu.import("resource://gre/modules/Log.jsm"); diff --git a/services/sync/modules/main.js b/services/sync/modules/main.js index e8e705e72..488a2594b 100644 --- a/services/sync/modules/main.js +++ b/services/sync/modules/main.js @@ -6,7 +6,7 @@ this.EXPORTED_SYMBOLS = ['Weave']; this.Weave = {}; Components.utils.import("resource://services-sync/constants.js", Weave); -var lazies = { +let lazies = { "jpakeclient.js": ["JPAKEClient", "SendCredentialsController"], "notifications.js": ["Notifications", "Notification", "NotificationButton"], "service.js": ["Service"], @@ -15,14 +15,12 @@ var lazies = { }; function lazyImport(module, dest, props) { - function getter(prop) { - return function() { - let ns = {}; - Components.utils.import(module, ns); - delete dest[prop]; - return dest[prop] = ns[prop]; - }; - } + function getter(prop) function() { + let ns = {}; + Components.utils.import(module, ns); + delete dest[prop]; + return dest[prop] = ns[prop]; + }; props.forEach(function (prop) { dest.__defineGetter__(prop, getter(prop)); }); } diff --git a/services/sync/modules/notifications.js b/services/sync/modules/notifications.js index 72187a4ce..1ee24f2cd 100644 --- a/services/sync/modules/notifications.js +++ b/services/sync/modules/notifications.js @@ -4,10 +4,10 @@ this.EXPORTED_SYMBOLS = ["Notifications", "Notification", "NotificationButton"]; -var Cc = Components.classes; -var Ci = Components.interfaces; -var Cr = Components.results; -var Cu = Components.utils; +const Cc = Components.classes; +const Ci = Components.interfaces; +const Cr = Components.results; +const Cu = Components.utils; Cu.import("resource://services-common/observers.js"); Cu.import("resource://gre/modules/Log.jsm"); diff --git a/services/sync/modules/policies.js b/services/sync/modules/policies.js index a3933426d..d799cb235 100644 --- a/services/sync/modules/policies.js +++ b/services/sync/modules/policies.js @@ -7,26 +7,16 @@ this.EXPORTED_SYMBOLS = [ "SyncScheduler", ]; -var {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components; +const {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components; Cu.import("resource://gre/modules/Log.jsm"); Cu.import("resource://services-sync/constants.js"); Cu.import("resource://services-sync/engines.js"); Cu.import("resource://services-sync/util.js"); Cu.import("resource://services-common/logmanager.js"); -Cu.import("resource://services-common/async.js"); XPCOMUtils.defineLazyModuleGetter(this, "Status", "resource://services-sync/status.js"); -XPCOMUtils.defineLazyModuleGetter(this, "AddonManager", - "resource://gre/modules/AddonManager.jsm"); - -// Get the value for an interval that's stored in preferences. To save users -// from themselves (and us from them!) the minimum time they can specify -// is 60s. -function getThrottledIntervalPreference(prefName) { - return Math.max(Svc.Prefs.get(prefName), 60) * 1000; -} this.SyncScheduler = function SyncScheduler(service) { this.service = service; @@ -55,12 +45,12 @@ SyncScheduler.prototype = { let part = service.fxAccountsEnabled ? "fxa" : "sync11"; let prefSDInterval = "scheduler." + part + ".singleDeviceInterval"; - this.singleDeviceInterval = getThrottledIntervalPreference(prefSDInterval); + this.singleDeviceInterval = Svc.Prefs.get(prefSDInterval) * 1000; - this.idleInterval = getThrottledIntervalPreference("scheduler.idleInterval"); - this.activeInterval = getThrottledIntervalPreference("scheduler.activeInterval"); - this.immediateInterval = getThrottledIntervalPreference("scheduler.immediateInterval"); - this.eolInterval = getThrottledIntervalPreference("scheduler.eolInterval"); + this.idleInterval = Svc.Prefs.get("scheduler.idleInterval") * 1000; + this.activeInterval = Svc.Prefs.get("scheduler.activeInterval") * 1000; + this.immediateInterval = Svc.Prefs.get("scheduler.immediateInterval") * 1000; + this.eolInterval = Svc.Prefs.get("scheduler.eolInterval") * 1000; // A user is non-idle on startup by default. this.idle = false; @@ -71,40 +61,20 @@ SyncScheduler.prototype = { }, // nextSync is in milliseconds, but prefs can't hold that much - get nextSync() { - return Svc.Prefs.get("nextSync", 0) * 1000; - }, - set nextSync(value) { - Svc.Prefs.set("nextSync", Math.floor(value / 1000)); - }, + get nextSync() Svc.Prefs.get("nextSync", 0) * 1000, + set nextSync(value) Svc.Prefs.set("nextSync", Math.floor(value / 1000)), - get syncInterval() { - return Svc.Prefs.get("syncInterval", this.singleDeviceInterval); - }, - set syncInterval(value) { - Svc.Prefs.set("syncInterval", value); - }, + get syncInterval() Svc.Prefs.get("syncInterval", this.singleDeviceInterval), + set syncInterval(value) Svc.Prefs.set("syncInterval", value), - get syncThreshold() { - return Svc.Prefs.get("syncThreshold", SINGLE_USER_THRESHOLD); - }, - set syncThreshold(value) { - Svc.Prefs.set("syncThreshold", value); - }, + get syncThreshold() Svc.Prefs.get("syncThreshold", SINGLE_USER_THRESHOLD), + set syncThreshold(value) Svc.Prefs.set("syncThreshold", value), - get globalScore() { - return Svc.Prefs.get("globalScore", 0); - }, - set globalScore(value) { - Svc.Prefs.set("globalScore", value); - }, + get globalScore() Svc.Prefs.get("globalScore", 0), + set globalScore(value) Svc.Prefs.set("globalScore", value), - get numClients() { - return Svc.Prefs.get("numClients", 0); - }, - set numClients(value) { - Svc.Prefs.set("numClients", value); - }, + get numClients() Svc.Prefs.get("numClients", 0), + set numClients(value) Svc.Prefs.set("numClients", value), init: function init() { this._log.level = Log.Level[Svc.Prefs.get("log.logger.service.main")]; @@ -249,10 +219,7 @@ SyncScheduler.prototype = { this.setDefaults(); try { Svc.Idle.removeIdleObserver(this, Svc.Prefs.get("scheduler.idleTime")); - } catch (ex) { - if (ex.result != Cr.NS_ERROR_FAILURE) { - throw ex; - } + } catch (ex if (ex.result == Cr.NS_ERROR_FAILURE)) { // In all likelihood we didn't have an idle observer registered yet. // It's all good. } @@ -285,11 +252,10 @@ SyncScheduler.prototype = { case "wake_notification": this._log.debug("Woke from sleep."); Utils.nextTick(() => { - // Trigger a sync if we have multiple clients. We give it 5 seconds - // incase the network is still in the process of coming back up. + // Trigger a sync if we have multiple clients. if (this.numClients > 1) { - this._log.debug("More than 1 client. Will sync in 5s."); - this.scheduleNextSync(5000); + this._log.debug("More than 1 client. Syncing."); + this.scheduleNextSync(0); } }); break; @@ -531,6 +497,45 @@ SyncScheduler.prototype = { this.syncTimer.clear(); }, + /** + * Prevent new syncs from starting. This is used by the FxA migration code + * where we can't afford to have a sync start partway through the migration. + * To handle the edge-case of a sync starting and not stopping, we store + * this state in a pref, so on the next startup we remain blocked (and thus + * sync will never start) so the migration can complete. + * + * As a safety measure, we only block for some period of time, and after + * that it will automatically unblock. This ensures that if things go + * really pear-shaped and we never end up calling unblockSync() we haven't + * completely broken the world. + */ + blockSync: function(until = null) { + if (!until) { + until = Date.now() + DEFAULT_BLOCK_PERIOD; + } + // until is specified in ms, but Prefs can't hold that much + Svc.Prefs.set("scheduler.blocked-until", Math.floor(until / 1000)); + }, + + unblockSync: function() { + Svc.Prefs.reset("scheduler.blocked-until"); + // the migration code should be ready to roll, so resume normal operations. + this.checkSyncStatus(); + }, + + get isBlocked() { + let until = Svc.Prefs.get("scheduler.blocked-until"); + if (until === undefined) { + return false; + } + if (until <= Math.floor(Date.now() / 1000)) { + // we were previously blocked but the time has expired. + Svc.Prefs.reset("scheduler.blocked-until"); + return false; + } + // we remain blocked. + return true; + }, }; this.ErrorHandler = function ErrorHandler(service) { @@ -570,10 +575,7 @@ ErrorHandler.prototype = { root.level = Log.Level[Svc.Prefs.get("log.rootLogger")]; let logs = ["Sync", "FirefoxAccounts", "Hawk", "Common.TokenServerClient", - "Sync.SyncMigration", "browserwindow.syncui", - "Services.Common.RESTRequest", "Services.Common.RESTRequest", - "BookmarkSyncUtils" - ]; + "Sync.SyncMigration"]; this._logManager = new LogManager(Svc.Prefs, logs, "sync"); }, @@ -590,25 +592,17 @@ ErrorHandler.prototype = { this._log.debug(data + " failed to apply some records."); } break; - case "weave:engine:sync:error": { + case "weave:engine:sync:error": let exception = subject; // exception thrown by engine's sync() method let engine_name = data; // engine name that threw the exception this.checkServerError(exception); Status.engines = [engine_name, exception.failureCode || ENGINE_UNKNOWN_FAIL]; - if (Async.isShutdownException(exception)) { - this._log.debug(engine_name + " was interrupted due to the application shutting down"); - } else { - this._log.debug(engine_name + " failed", exception); - Services.telemetry.getKeyedHistogramById("WEAVE_ENGINE_SYNC_ERRORS") - .add(engine_name); - } + this._log.debug(engine_name + " failed: " + Utils.exceptionStr(exception)); break; - } case "weave:service:login:error": - this._log.error("Sync encountered a login error"); - this.resetFileLog(); + this.resetFileLog(this._logManager.REASON_ERROR); if (this.shouldReportError()) { this.notifyOnNextTick("weave:ui:login:error"); @@ -618,23 +612,12 @@ ErrorHandler.prototype = { this.dontIgnoreErrors = false; break; - case "weave:service:sync:error": { + case "weave:service:sync:error": if (Status.sync == CREDENTIALS_CHANGED) { this.service.logout(); } - let exception = subject; - if (Async.isShutdownException(exception)) { - // If we are shutting down we just log the fact, attempt to flush - // the log file and get out of here! - this._log.error("Sync was interrupted due to the application shutting down"); - this.resetFileLog(); - break; - } - - // Not a shutdown related exception... - this._log.error("Sync encountered an error", exception); - this.resetFileLog(); + this.resetFileLog(this._logManager.REASON_ERROR); if (this.shouldReportError()) { this.notifyOnNextTick("weave:ui:sync:error"); @@ -644,7 +627,6 @@ ErrorHandler.prototype = { this.dontIgnoreErrors = false; break; - } case "weave:service:sync:finish": this._log.trace("Status.service is " + Status.service); @@ -660,8 +642,8 @@ ErrorHandler.prototype = { } if (Status.service == SYNC_FAILED_PARTIAL) { - this._log.error("Some engines did not sync correctly."); - this.resetFileLog(); + this._log.debug("Some engines did not sync correctly."); + this.resetFileLog(this._logManager.REASON_ERROR); if (this.shouldReportError()) { this.dontIgnoreErrors = false; @@ -669,7 +651,7 @@ ErrorHandler.prototype = { break; } } else { - this.resetFileLog(); + this.resetFileLog(this._logManager.REASON_SUCCESS); } this.dontIgnoreErrors = false; this.notifyOnNextTick("weave:ui:sync:finish"); @@ -696,52 +678,22 @@ ErrorHandler.prototype = { Utils.nextTick(this.service.sync, this.service); }, - _dumpAddons: function _dumpAddons() { - // Just dump the items that sync may be concerned with. Specifically, - // active extensions that are not hidden. - let addonPromise = new Promise(resolve => { - try { - AddonManager.getAddonsByTypes(["extension"], resolve); - } catch (e) { - this._log.warn("Failed to dump addons", e) - resolve([]) - } - }); - - return addonPromise.then(addons => { - let relevantAddons = addons.filter(x => x.isActive && !x.hidden); - this._log.debug("Addons installed", relevantAddons.length); - for (let addon of relevantAddons) { - this._log.debug(" - ${name}, version ${version}, id ${id}", addon); - } - }); - }, - /** * Generate a log file for the sync that just completed * and refresh the input & output streams. + * + * @param reason + * A constant from the LogManager that indicates the reason for the + * reset. */ - resetFileLog: function resetFileLog() { - let onComplete = logType => { + resetFileLog: function resetFileLog(reason) { + let onComplete = () => { Svc.Obs.notify("weave:service:reset-file-log"); this._log.trace("Notified: " + Date.now()); - if (logType == this._logManager.ERROR_LOG_WRITTEN) { - Cu.reportError("Sync encountered an error - see about:sync-log for the log file."); - } }; - - // If we're writing an error log, dump extensions that may be causing problems. - let beforeResetLog; - if (this._logManager.sawError) { - beforeResetLog = this._dumpAddons(); - } else { - beforeResetLog = Promise.resolve(); - } // Note we do not return the promise here - the caller doesn't need to wait // for this to complete. - beforeResetLog - .then(() => this._logManager.resetFileLog()) - .then(onComplete, onComplete); + this._logManager.resetFileLog(reason).then(onComplete, onComplete); }, /** @@ -775,9 +727,6 @@ ErrorHandler.prototype = { } }, - // A function to indicate if Sync errors should be "reported" - which in this - // context really means "should be notify observers of an error" - but note - // that since bug 1180587, no one is going to surface an error to the user. shouldReportError: function shouldReportError() { if (Status.login == MASTER_PASSWORD_LOCKED) { this._log.trace("shouldReportError: false (master password locked)."); @@ -817,12 +766,8 @@ ErrorHandler.prototype = { return false; } - - let result = ([Status.login, Status.sync].indexOf(SERVER_MAINTENANCE) == -1 && - [Status.login, Status.sync].indexOf(LOGIN_FAILED_NETWORK_ERROR) == -1); - this._log.trace("shouldReportError: ${result} due to login=${login}, sync=${sync}", - {result, login: Status.login, sync: Status.sync}); - return result; + return ([Status.login, Status.sync].indexOf(SERVER_MAINTENANCE) == -1 && + [Status.login, Status.sync].indexOf(LOGIN_FAILED_NETWORK_ERROR) == -1); }, get currentAlertMode() { @@ -925,7 +870,7 @@ ErrorHandler.prototype = { case 401: this.service.logout(); this._log.info("Got 401 response; resetting clusterURL."); - this.service.clusterURL = null; + Svc.Prefs.reset("clusterURL"); let delay = 0; if (Svc.Prefs.get("lastSyncReassigned")) { diff --git a/services/sync/modules/record.js b/services/sync/modules/record.js index f7a69d9ef..04ccd2dd2 100644 --- a/services/sync/modules/record.js +++ b/services/sync/modules/record.js @@ -10,10 +10,10 @@ this.EXPORTED_SYMBOLS = [ "Collection", ]; -var Cc = Components.classes; -var Ci = Components.interfaces; -var Cr = Components.results; -var Cu = Components.utils; +const Cc = Components.classes; +const Ci = Components.interfaces; +const Cr = Components.results; +const Cu = Components.utils; const CRYPTO_COLLECTION = "crypto"; const KEYS_WBO = "keys"; @@ -23,7 +23,6 @@ Cu.import("resource://services-sync/constants.js"); Cu.import("resource://services-sync/keys.js"); Cu.import("resource://services-sync/resource.js"); Cu.import("resource://services-sync/util.js"); -Cu.import("resource://services-common/async.js"); this.WBORecord = function WBORecord(collection, id) { this.data = {}; @@ -86,7 +85,7 @@ WBORecord.prototype = { toJSON: function toJSON() { // Copy fields from data to be stringified, making sure payload is a string let obj = {}; - for (let [key, val] of Object.entries(this.data)) + for (let [key, val] in Iterator(this.data)) obj[key] = key == "payload" ? JSON.stringify(val) : val; if (this.ttl) obj.ttl = this.ttl; @@ -196,9 +195,7 @@ CryptoWrapper.prototype = { }, // The custom setter below masks the parent's getter, so explicitly call it :( - get id() { - return WBORecord.prototype.__lookupGetter__("id").call(this); - }, + get id() WBORecord.prototype.__lookupGetter__("id").call(this), // Keep both plaintext and encrypted versions of the id to verify integrity set id(val) { @@ -238,11 +235,8 @@ RecordManager.prototype = { record.deserialize(this.response); return this.set(url, record); - } catch (ex) { - if (Async.isShutdownException(ex)) { - throw ex; - } - this._log.debug("Failed to import record", ex); + } catch(ex) { + this._log.debug("Failed to import record: " + Utils.exceptionStr(ex)); return null; } }, @@ -281,10 +275,10 @@ RecordManager.prototype = { * You can update this thing simply by giving it /info/collections. It'll * use the last modified time to bring itself up to date. */ -this.CollectionKeyManager = function CollectionKeyManager(lastModified, default_, collections) { - this.lastModified = lastModified || 0; - this._default = default_ || null; - this._collections = collections || {}; +this.CollectionKeyManager = function CollectionKeyManager() { + this.lastModified = 0; + this._collections = {}; + this._default = null; this._log = Log.repository.getLogger("Sync.CollectionKeyManager"); } @@ -293,19 +287,6 @@ this.CollectionKeyManager = function CollectionKeyManager(lastModified, default_ // Note that the last modified time needs to be preserved. CollectionKeyManager.prototype = { - /** - * Generate a new CollectionKeyManager that has the same attributes - * as this one. - */ - clone() { - const newCollections = {}; - for (let c in this._collections) { - newCollections[c] = this._collections[c]; - } - - return new CollectionKeyManager(this.lastModified, this._default, newCollections); - }, - // Return information about old vs new keys: // * same: true if two collections are equal // * changed: an array of collection names that changed. @@ -328,7 +309,7 @@ CollectionKeyManager.prototype = { // Return a sorted, unique array. changed.sort(); let last; - changed = changed.filter(x => (x != last) && (last = x)); + changed = [x for each (x in changed) if ((x != last) && (last = x))]; return {same: changed.length == 0, changed: changed}; }, @@ -374,15 +355,15 @@ CollectionKeyManager.prototype = { /** * Create a WBO for the current keys. */ - asWBO: function(collection, id) { - return this._makeWBO(this._collections, this._default); - }, + asWBO: function(collection, id) + this._makeWBO(this._collections, this._default), /** * Compute a new default key, and new keys for any specified collections. */ newKeys: function(collections) { - let newDefaultKeyBundle = this.newDefaultKeyBundle(); + let newDefaultKey = new BulkKeyBundle(DEFAULT_KEYBUNDLE_NAME); + newDefaultKey.generateRandom(); let newColls = {}; if (collections) { @@ -392,7 +373,7 @@ CollectionKeyManager.prototype = { newColls[c] = b; }); } - return [newDefaultKeyBundle, newColls]; + return [newDefaultKey, newColls]; }, /** @@ -406,57 +387,6 @@ CollectionKeyManager.prototype = { return this._makeWBO(newColls, newDefaultKey); }, - /** - * Create a new default key. - * - * @returns {BulkKeyBundle} - */ - newDefaultKeyBundle() { - const key = new BulkKeyBundle(DEFAULT_KEYBUNDLE_NAME); - key.generateRandom(); - return key; - }, - - /** - * Create a new default key and store it as this._default, since without one you cannot use setContents. - */ - generateDefaultKey() { - this._default = this.newDefaultKeyBundle(); - }, - - /** - * Return true if keys are already present for each of the given - * collections. - */ - hasKeysFor(collections) { - // We can't use filter() here because sometimes collections is an iterator. - for (let collection of collections) { - if (!this._collections[collection]) { - return false; - } - } - return true; - }, - - /** - * Return a new CollectionKeyManager that has keys for each of the - * given collections (creating new ones for collections where we - * don't already have keys). - */ - ensureKeysFor(collections) { - const newKeys = Object.assign({}, this._collections); - for (let c of collections) { - if (newKeys[c]) { - continue; // don't replace existing keys - } - - const b = new BulkKeyBundle(c); - b.generateRandom(); - newKeys[c] = b; - } - return new CollectionKeyManager(this.lastModified, this._default, newKeys); - }, - // Take the fetched info/collections WBO, checking the change // time of the crypto collection. updateNeeded: function(info_collections) { @@ -487,6 +417,9 @@ CollectionKeyManager.prototype = { // setContents: function setContents(payload, modified) { + if (!modified) + throw "No modified time provided to setContents."; + let self = this; this._log.info("Setting collection keys contents. Our last modified: " + @@ -516,7 +449,9 @@ CollectionKeyManager.prototype = { if (v) { let keyObj = new BulkKeyBundle(k); keyObj.keyPairB64 = v; - newCollections[k] = keyObj; + if (keyObj) { + newCollections[k] = keyObj; + } } } } @@ -527,11 +462,8 @@ CollectionKeyManager.prototype = { let sameColls = collComparison.same; if (sameDefault && sameColls) { - self._log.info("New keys are the same as our old keys!"); - if (modified) { - self._log.info("Bumped local modified time."); - self.lastModified = modified; - } + self._log.info("New keys are the same as our old keys! Bumped local modified time."); + self.lastModified = modified; return false; } @@ -543,10 +475,8 @@ CollectionKeyManager.prototype = { this._collections = newCollections; // Always trust the server. - if (modified) { - self._log.info("Bumping last modified to " + modified); - self.lastModified = modified; - } + self._log.info("Bumping last modified to " + modified); + self.lastModified = modified; return sameDefault ? collComparison.changed : true; }, @@ -594,12 +524,6 @@ this.Collection = function Collection(uri, recordObj, service) { this._older = 0; this._newer = 0; this._data = []; - // optional members used by batch upload operations. - this._batch = null; - this._commit = false; - // Used for batch download operations -- note that this is explicitly an - // opaque value and not (necessarily) a number. - this._offset = null; } Collection.prototype = { __proto__: Resource.prototype, @@ -623,12 +547,6 @@ Collection.prototype = { args.push("ids=" + this.ids); if (this.limit > 0 && this.limit != Infinity) args.push("limit=" + this.limit); - if (this._batch) - args.push("batch=" + encodeURIComponent(this._batch)); - if (this._commit) - args.push("commit=true"); - if (this._offset) - args.push("offset=" + encodeURIComponent(this._offset)); this.uri.query = (args.length > 0)? '?' + args.join('&') : ''; }, @@ -641,14 +559,14 @@ Collection.prototype = { }, // Apply the action to a certain set of ids - get ids() { return this._ids; }, + get ids() this._ids, set ids(value) { this._ids = value; this._rebuildURL(); }, // Limit how many records to get - get limit() { return this._limit; }, + get limit() this._limit, set limit(value) { this._limit = value; this._rebuildURL(); @@ -678,100 +596,12 @@ Collection.prototype = { this._rebuildURL(); }, - get offset() { return this._offset; }, - set offset(value) { - this._offset = value; - this._rebuildURL(); - }, - - // Set information about the batch for this request. - get batch() { return this._batch; }, - set batch(value) { - this._batch = value; - this._rebuildURL(); - }, - - get commit() { return this._commit; }, - set commit(value) { - this._commit = value && true; - this._rebuildURL(); + pushData: function Coll_pushData(data) { + this._data.push(data); }, - // Similar to get(), but will page through the items `batchSize` at a time, - // deferring calling the record handler until we've gotten them all. - // - // Returns the last response processed, and doesn't run the record handler - // on any items if a non-success status is received while downloading the - // records (or if a network error occurs). - getBatched(batchSize = DEFAULT_DOWNLOAD_BATCH_SIZE) { - let totalLimit = Number(this.limit) || Infinity; - if (batchSize <= 0 || batchSize >= totalLimit) { - // Invalid batch sizes should arguably be an error, but they're easy to handle - return this.get(); - } - - if (!this.full) { - throw new Error("getBatched is unimplemented for guid-only GETs"); - } - - // _onComplete and _onProgress are reset after each `get` by AsyncResource. - // We overwrite _onRecord to something that stores the data in an array - // until the end. - let { _onComplete, _onProgress, _onRecord } = this; - let recordBuffer = []; - let resp; - try { - this._onRecord = r => recordBuffer.push(r); - let lastModifiedTime; - this.limit = batchSize; - - do { - this._onProgress = _onProgress; - this._onComplete = _onComplete; - if (batchSize + recordBuffer.length > totalLimit) { - this.limit = totalLimit - recordBuffer.length; - } - this._log.trace("Performing batched GET", { limit: this.limit, offset: this.offset }); - // Actually perform the request - resp = this.get(); - if (!resp.success) { - break; - } - - // Initialize last modified, or check that something broken isn't happening. - let lastModified = resp.headers["x-last-modified"]; - if (!lastModifiedTime) { - lastModifiedTime = lastModified; - this.setHeader("X-If-Unmodified-Since", lastModified); - } else if (lastModified != lastModifiedTime) { - // Should be impossible -- We'd get a 412 in this case. - throw new Error("X-Last-Modified changed in the middle of a download batch! " + - `${lastModified} => ${lastModifiedTime}`) - } - - // If this is missing, we're finished. - this.offset = resp.headers["x-weave-next-offset"]; - } while (this.offset && totalLimit > recordBuffer.length); - } finally { - // Ensure we undo any temporary state so that subsequent calls to get() - // or getBatched() work properly. We do this before calling the record - // handler so that we can more convincingly pretend to be a normal get() - // call. Note: we're resetting these to the values they had before this - // function was called. - this._onRecord = _onRecord; - this._limit = totalLimit; - this._offset = null; - delete this._headers["x-if-unmodified-since"]; - this._rebuildURL(); - } - if (resp.success && Async.checkAppReady()) { - // call the original _onRecord (e.g. the user supplied record handler) - // for each record we've stored - for (let record of recordBuffer) { - this._onRecord(record); - } - } - return resp; + clearRecords: function Coll_clearRecords() { + this._data = []; }, set recordHandler(onRecord) { @@ -781,8 +611,6 @@ Collection.prototype = { // Switch to newline separated records for incremental parsing coll.setHeader("Accept", "application/newlines"); - this._onRecord = onRecord; - this._onProgress = function() { let newline; while ((newline = this._data.indexOf("\n")) > 0) { @@ -793,247 +621,8 @@ Collection.prototype = { // Deserialize a record from json and give it to the callback let record = new coll._recordObj(); record.deserialize(json); - coll._onRecord(record); + onRecord(record); } }; }, - - // This object only supports posting via the postQueue object. - post() { - throw new Error("Don't directly post to a collection - use newPostQueue instead"); - }, - - newPostQueue(log, timestamp, postCallback) { - let poster = (data, headers, batch, commit) => { - this.batch = batch; - this.commit = commit; - for (let [header, value] of headers) { - this.setHeader(header, value); - } - return Resource.prototype.post.call(this, data); - } - let getConfig = (name, defaultVal) => { - if (this._service.serverConfiguration && this._service.serverConfiguration.hasOwnProperty(name)) { - return this._service.serverConfiguration[name]; - } - return defaultVal; - } - - let config = { - max_post_bytes: getConfig("max_post_bytes", MAX_UPLOAD_BYTES), - max_post_records: getConfig("max_post_records", MAX_UPLOAD_RECORDS), - - max_batch_bytes: getConfig("max_total_bytes", Infinity), - max_batch_records: getConfig("max_total_records", Infinity), - } - - // Handle config edge cases - if (config.max_post_records <= 0) { config.max_post_records = MAX_UPLOAD_RECORDS; } - if (config.max_batch_records <= 0) { config.max_batch_records = Infinity; } - if (config.max_post_bytes <= 0) { config.max_post_bytes = MAX_UPLOAD_BYTES; } - if (config.max_batch_bytes <= 0) { config.max_batch_bytes = Infinity; } - - // Max size of BSO payload is 256k. This assumes at most 4k of overhead, - // which sounds like plenty. If the server says it can't handle this, we - // might have valid records we can't sync, so we give up on syncing. - let requiredMax = 260 * 1024; - if (config.max_post_bytes < requiredMax) { - this._log.error("Server configuration max_post_bytes is too low", config); - throw new Error("Server configuration max_post_bytes is too low"); - } - - return new PostQueue(poster, timestamp, config, log, postCallback); - }, }; - -/* A helper to manage the posting of records while respecting the various - size limits. - - This supports the concept of a server-side "batch". The general idea is: - * We queue as many records as allowed in memory, then make a single POST. - * This first POST (optionally) gives us a batch ID, which we use for - all subsequent posts, until... - * At some point we hit a batch-maximum, and jump through a few hoops to - commit the current batch (ie, all previous POSTs) and start a new one. - * Eventually commit the final batch. - - In most cases we expect there to be exactly 1 batch consisting of possibly - multiple POSTs. -*/ -function PostQueue(poster, timestamp, config, log, postCallback) { - // The "post" function we should use when it comes time to do the post. - this.poster = poster; - this.log = log; - - // The config we use. We expect it to have fields "max_post_records", - // "max_batch_records", "max_post_bytes", and "max_batch_bytes" - this.config = config; - - // The callback we make with the response when we do get around to making the - // post (which could be during any of the enqueue() calls or the final flush()) - // This callback may be called multiple times and must not add new items to - // the queue. - // The second argument passed to this callback is a boolean value that is true - // if we're in the middle of a batch, and false if either the batch is - // complete, or it's a post to a server that does not understand batching. - this.postCallback = postCallback; - - // The string where we are capturing the stringified version of the records - // queued so far. It will always be invalid JSON as it is always missing the - // closing bracket. - this.queued = ""; - - // The number of records we've queued so far but are yet to POST. - this.numQueued = 0; - - // The number of records/bytes we've processed in previous POSTs for our - // current batch. Does *not* include records currently queued for the next POST. - this.numAlreadyBatched = 0; - this.bytesAlreadyBatched = 0; - - // The ID of our current batch. Can be undefined (meaning we are yet to make - // the first post of a patch, so don't know if we have a batch), null (meaning - // we've made the first post but the server response indicated no batching - // semantics), otherwise we have made the first post and it holds the batch ID - // returned from the server. - this.batchID = undefined; - - // Time used for X-If-Unmodified-Since -- should be the timestamp from the last GET. - this.lastModified = timestamp; -} - -PostQueue.prototype = { - enqueue(record) { - // We want to ensure the record has a .toJSON() method defined - even - // though JSON.stringify() would implicitly call it, the stringify might - // still work even if it isn't defined, which isn't what we want. - let jsonRepr = record.toJSON(); - if (!jsonRepr) { - throw new Error("You must only call this with objects that explicitly support JSON"); - } - let bytes = JSON.stringify(jsonRepr); - - // Do a flush if we can't add this record without exceeding our single-request - // limits, or without exceeding the total limit for a single batch. - let newLength = this.queued.length + bytes.length + 2; // extras for leading "[" / "," and trailing "]" - - let maxAllowedBytes = Math.min(256 * 1024, this.config.max_post_bytes); - - let postSizeExceeded = this.numQueued >= this.config.max_post_records || - newLength >= maxAllowedBytes; - - let batchSizeExceeded = (this.numQueued + this.numAlreadyBatched) >= this.config.max_batch_records || - (newLength + this.bytesAlreadyBatched) >= this.config.max_batch_bytes; - - let singleRecordTooBig = bytes.length + 2 > maxAllowedBytes; - - if (postSizeExceeded || batchSizeExceeded) { - this.log.trace(`PostQueue flushing due to postSizeExceeded=${postSizeExceeded}, batchSizeExceeded=${batchSizeExceeded}` + - `, max_batch_bytes: ${this.config.max_batch_bytes}, max_post_bytes: ${this.config.max_post_bytes}`); - - if (singleRecordTooBig) { - return { enqueued: false, error: new Error("Single record too large to submit to server") }; - } - - // We need to write the queue out before handling this one, but we only - // commit the batch (and thus start a new one) if the batch is full. - // Note that if a single record is too big for the batch or post, then - // the batch may be empty, and so we don't flush in that case. - if (this.numQueued) { - this.flush(batchSizeExceeded || singleRecordTooBig); - } - } - // Either a ',' or a '[' depending on whether this is the first record. - this.queued += this.numQueued ? "," : "["; - this.queued += bytes; - this.numQueued++; - return { enqueued: true }; - }, - - flush(finalBatchPost) { - if (!this.queued) { - // nothing queued - we can't be in a batch, and something has gone very - // bad if we think we are. - if (this.batchID) { - throw new Error(`Flush called when no queued records but we are in a batch ${this.batchID}`); - } - return; - } - // the batch query-param and headers we'll send. - let batch; - let headers = []; - if (this.batchID === undefined) { - // First commit in a (possible) batch. - batch = "true"; - } else if (this.batchID) { - // We have an existing batch. - batch = this.batchID; - } else { - // Not the first post and we know we have no batch semantics. - batch = null; - } - - headers.push(["x-if-unmodified-since", this.lastModified]); - - this.log.info(`Posting ${this.numQueued} records of ${this.queued.length+1} bytes with batch=${batch}`); - let queued = this.queued + "]"; - if (finalBatchPost) { - this.bytesAlreadyBatched = 0; - this.numAlreadyBatched = 0; - } else { - this.bytesAlreadyBatched += queued.length; - this.numAlreadyBatched += this.numQueued; - } - this.queued = ""; - this.numQueued = 0; - let response = this.poster(queued, headers, batch, !!(finalBatchPost && this.batchID !== null)); - - if (!response.success) { - this.log.trace("Server error response during a batch", response); - // not clear what we should do here - we expect the consumer of this to - // abort by throwing in the postCallback below. - return this.postCallback(response, !finalBatchPost); - } - - if (finalBatchPost) { - this.log.trace("Committed batch", this.batchID); - this.batchID = undefined; // we are now in "first post for the batch" state. - this.lastModified = response.headers["x-last-modified"]; - return this.postCallback(response, false); - } - - if (response.status != 202) { - if (this.batchID) { - throw new Error("Server responded non-202 success code while a batch was in progress"); - } - this.batchID = null; // no batch semantics are in place. - this.lastModified = response.headers["x-last-modified"]; - return this.postCallback(response, false); - } - - // this response is saying the server has batch semantics - we should - // always have a batch ID in the response. - let responseBatchID = response.obj.batch; - this.log.trace("Server responsed 202 with batch", responseBatchID); - if (!responseBatchID) { - this.log.error("Invalid server response: 202 without a batch ID", response); - throw new Error("Invalid server response: 202 without a batch ID"); - } - - if (this.batchID === undefined) { - this.batchID = responseBatchID; - if (!this.lastModified) { - this.lastModified = response.headers["x-last-modified"]; - if (!this.lastModified) { - throw new Error("Batch response without x-last-modified"); - } - } - } - - if (this.batchID != responseBatchID) { - throw new Error(`Invalid client/server batch state - client has ${this.batchID}, server has ${responseBatchID}`); - } - - this.postCallback(response, true); - }, -} diff --git a/services/sync/modules/resource.js b/services/sync/modules/resource.js index bf7066b9f..1c2a67b90 100644 --- a/services/sync/modules/resource.js +++ b/services/sync/modules/resource.js @@ -7,13 +7,12 @@ this.EXPORTED_SYMBOLS = [ "Resource" ]; -var Cc = Components.classes; -var Ci = Components.interfaces; -var Cr = Components.results; -var Cu = Components.utils; +const Cc = Components.classes; +const Ci = Components.interfaces; +const Cr = Components.results; +const Cu = Components.utils; Cu.import("resource://gre/modules/Preferences.jsm"); -Cu.import("resource://gre/modules/NetUtil.jsm"); Cu.import("resource://services-common/async.js"); Cu.import("resource://gre/modules/Log.jsm"); Cu.import("resource://services-common/observers.js"); @@ -74,6 +73,20 @@ AsyncResource.prototype = { */ authenticator: null, + // The string to use as the base User-Agent in Sync requests. + // These strings will look something like + // + // Firefox/4.0 FxSync/1.8.0.20100101.mobile + // + // or + // + // Firefox Aurora/5.0a1 FxSync/1.9.0.20110409.desktop + // + _userAgent: + Services.appinfo.name + "/" + Services.appinfo.version + // Product. + " FxSync/" + WEAVE_VERSION + "." + // Sync. + Services.appinfo.appBuildID + ".", // Build. + // Wait 5 minutes before killing a request. ABORT_TIMEOUT: 300000, @@ -121,9 +134,7 @@ AsyncResource.prototype = { // // Get and set the data encapulated in the resource. _data: null, - get data() { - return this._data; - }, + get data() this._data, set data(value) { this._data = value; }, @@ -135,9 +146,16 @@ AsyncResource.prototype = { // to obtain a request channel. // _createRequest: function Res__createRequest(method) { - let channel = NetUtil.newChannel({uri: this.spec, loadUsingSystemPrincipal: true}) - .QueryInterface(Ci.nsIRequest) - .QueryInterface(Ci.nsIHttpChannel); + let channel = Services.io.newChannel2(this.spec, + null, + null, + null, // aLoadingNode + Services.scriptSecurityManager.getSystemPrincipal(), + null, // aTriggeringPrincipal + Ci.nsILoadInfo.SEC_NORMAL, + Ci.nsIContentPolicy.TYPE_OTHER) + .QueryInterface(Ci.nsIRequest) + .QueryInterface(Ci.nsIHttpChannel); channel.loadFlags |= DEFAULT_LOAD_FLAGS; @@ -147,7 +165,8 @@ AsyncResource.prototype = { // Compose a UA string fragment from the various available identifiers. if (Svc.Prefs.get("sendVersionInfo", true)) { - channel.setRequestHeader("user-agent", Utils.userAgent, false); + let ua = this._userAgent + Svc.Prefs.get("client.type", "desktop"); + channel.setRequestHeader("user-agent", ua, false); } let headers = this.headers; @@ -155,7 +174,7 @@ AsyncResource.prototype = { if (this.authenticator) { let result = this.authenticator(this, method); if (result && result.headers) { - for (let [k, v] of Object.entries(result.headers)) { + for (let [k, v] in Iterator(result.headers)) { headers[k.toLowerCase()] = v; } } @@ -163,7 +182,7 @@ AsyncResource.prototype = { this._log.debug("No authenticator found."); } - for (let [key, value] of Object.entries(headers)) { + for (let [key, value] in Iterator(headers)) { if (key == 'authorization') this._log.trace("HTTP Header " + key + ": ***** (suppressed)"); else @@ -209,10 +228,10 @@ AsyncResource.prototype = { this._log, this.ABORT_TIMEOUT); channel.requestMethod = action; try { - channel.asyncOpen2(listener); + channel.asyncOpen(listener, null); } catch (ex) { - // asyncOpen2 can throw in a bunch of cases -- e.g., a forbidden port. - this._log.warn("Caught an error in asyncOpen2", ex); + // asyncOpen can throw in a bunch of cases -- e.g., a forbidden port. + this._log.warn("Caught an error in asyncOpen: " + CommonUtils.exceptionStr(ex)); CommonUtils.nextTick(callback.bind(this, ex)); } }, @@ -259,7 +278,9 @@ AsyncResource.prototype = { } catch(ex) { // Got a response, but an exception occurred during processing. // This shouldn't occur. - this._log.warn("Caught unexpected exception in _oncomplete", ex); + this._log.warn("Caught unexpected exception " + CommonUtils.exceptionStr(ex) + + " in _onComplete."); + this._log.debug(CommonUtils.stackTrace(ex)); } // Process headers. They can be empty, or the call can otherwise fail, so @@ -297,18 +318,16 @@ AsyncResource.prototype = { contentLength + "."); } } catch (ex) { - this._log.debug("Caught exception visiting headers in _onComplete", ex); + this._log.debug("Caught exception " + CommonUtils.exceptionStr(ex) + + " visiting headers in _onComplete."); + this._log.debug(CommonUtils.stackTrace(ex)); } let ret = new String(data); - ret.url = channel.URI.spec; ret.status = status; ret.success = success; ret.headers = headers; - if (!success) { - this._log.warn(`${action} request to ${ret.url} failed with status ${status}`); - } // Make a lazy getter to convert the json response into an object. // Note that this can cause a parse error to be thrown far away from the // actual fetch, so be warned! @@ -316,7 +335,7 @@ AsyncResource.prototype = { try { return JSON.parse(ret); } catch (ex) { - this._log.warn("Got exception parsing response body", ex); + this._log.warn("Got exception parsing response body: \"" + CommonUtils.exceptionStr(ex)); // Stringify to avoid possibly printing non-printable characters. this._log.debug("Parse fail: Response body starts: \"" + JSON.stringify((ret + "").slice(0, 100)) + @@ -384,12 +403,7 @@ Resource.prototype = { try { this._doRequest(action, data, callback); return Async.waitForSyncCallback(cb); - } catch (ex) { - if (Async.isShutdownException(ex)) { - throw ex; - } - this._log.warn("${action} request to ${url} failed: ${ex}", - { action, url: this.uri.spec, ex }); + } catch(ex) { // Combine the channel stack with this request stack. Need to create // a new error object for that. let error = Error(ex.message); @@ -527,7 +541,7 @@ ChannelListener.prototype = { siStream = Cc["@mozilla.org/scriptableinputstream;1"].createInstance(Ci.nsIScriptableInputStream); siStream.init(stream); } catch (ex) { - this._log.warn("Exception creating nsIScriptableInputStream", ex); + this._log.warn("Exception creating nsIScriptableInputStream." + CommonUtils.exceptionStr(ex)); this._log.debug("Parameters: " + req.URI.spec + ", " + stream + ", " + off + ", " + count); // Cannot proceed, so rethrow and allow the channel to cancel itself. throw ex; @@ -543,11 +557,9 @@ ChannelListener.prototype = { try { this._onProgress(); } catch (ex) { - if (Async.isShutdownException(ex)) { - throw ex; - } this._log.warn("Got exception calling onProgress handler during fetch of " - + req.URI.spec, ex); + + req.URI.spec); + this._log.debug(CommonUtils.exceptionStr(ex)); this._log.trace("Rethrowing; expect a failure code from the HTTP channel."); throw ex; } @@ -562,7 +574,7 @@ ChannelListener.prototype = { try { CommonUtils.namedTimer(this.abortRequest, this._timeout, this, "abortTimer"); } catch (ex) { - this._log.warn("Got exception extending abort timer", ex); + this._log.warn("Got exception extending abort timer: " + CommonUtils.exceptionStr(ex)); } }, @@ -656,14 +668,14 @@ ChannelNotificationListener.prototype = { } } } catch (ex) { - this._log.error("Error copying headers", ex); + this._log.error("Error copying headers: " + CommonUtils.exceptionStr(ex)); } // We let all redirects proceed. try { callback.onRedirectVerifyCallback(Cr.NS_OK); } catch (ex) { - this._log.error("onRedirectVerifyCallback threw!", ex); + this._log.error("onRedirectVerifyCallback threw!" + CommonUtils.exceptionStr(ex)); } } }; diff --git a/services/sync/modules/rest.js b/services/sync/modules/rest.js index 94c096dba..34382eed5 100644 --- a/services/sync/modules/rest.js +++ b/services/sync/modules/rest.js @@ -2,7 +2,7 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -var {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components; +const {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components; Cu.import("resource://gre/modules/Log.jsm"); Cu.import("resource://services-common/rest.js"); @@ -28,6 +28,21 @@ SyncStorageRequest.prototype = { _logName: "Sync.StorageRequest", /** + * The string to use as the base User-Agent in Sync requests. + * These strings will look something like + * + * Firefox/4.0 FxSync/1.8.0.20100101.mobile + * + * or + * + * Firefox Aurora/5.0a1 FxSync/1.9.0.20110409.desktop + */ + userAgent: + Services.appinfo.name + "/" + Services.appinfo.version + // Product. + " FxSync/" + WEAVE_VERSION + "." + // Sync. + Services.appinfo.appBuildID + ".", // Build. + + /** * Wait 5 minutes before killing a request. */ timeout: STORAGE_REQUEST_TIMEOUT, @@ -35,7 +50,8 @@ SyncStorageRequest.prototype = { dispatch: function dispatch(method, data, onComplete, onProgress) { // Compose a UA string fragment from the various available identifiers. if (Svc.Prefs.get("sendVersionInfo", true)) { - this.setHeader("user-agent", Utils.userAgent); + let ua = this.userAgent + Svc.Prefs.get("client.type", "desktop"); + this.setHeader("user-agent", ua); } if (this.authenticator) { diff --git a/services/sync/modules/service.js b/services/sync/modules/service.js index 32e047f53..637760b8f 100644 --- a/services/sync/modules/service.js +++ b/services/sync/modules/service.js @@ -4,10 +4,10 @@ this.EXPORTED_SYMBOLS = ["Service"]; -var Cc = Components.classes; -var Ci = Components.interfaces; -var Cr = Components.results; -var Cu = Components.utils; +const Cc = Components.classes; +const Ci = Components.interfaces; +const Cr = Components.results; +const Cu = Components.utils; // How long before refreshing the cluster const CLUSTER_BACKOFF = 5 * 60 * 1000; // 5 minutes @@ -21,6 +21,7 @@ const KEYS_WBO = "keys"; Cu.import("resource://gre/modules/Preferences.jsm"); Cu.import("resource://gre/modules/XPCOMUtils.jsm"); Cu.import("resource://gre/modules/Log.jsm"); +Cu.import("resource://services-common/utils.js"); Cu.import("resource://services-sync/constants.js"); Cu.import("resource://services-sync/engines.js"); Cu.import("resource://services-sync/engines/clients.js"); @@ -32,7 +33,6 @@ Cu.import("resource://services-sync/rest.js"); Cu.import("resource://services-sync/stages/enginesync.js"); Cu.import("resource://services-sync/stages/declined.js"); Cu.import("resource://services-sync/status.js"); -Cu.import("resource://services-sync/telemetry.js"); Cu.import("resource://services-sync/userapi.js"); Cu.import("resource://services-sync/util.js"); @@ -51,6 +51,15 @@ const STORAGE_INFO_TYPES = [INFO_COLLECTIONS, INFO_COLLECTION_COUNTS, INFO_QUOTA]; +// A structure mapping a (boolean) telemetry probe name to a preference name. +// The probe will record true if the pref is modified, false otherwise. +const TELEMETRY_CUSTOM_SERVER_PREFS = { + WEAVE_CUSTOM_LEGACY_SERVER_CONFIGURATION: "services.sync.serverURL", + WEAVE_CUSTOM_FXA_SERVER_CONFIGURATION: "identity.fxaccounts.auth.uri", + WEAVE_CUSTOM_TOKEN_SERVER_CONFIGURATION: "services.sync.tokenServerURI", +}; + + function Sync11Service() { this._notify = Utils.notify("weave:service:"); } @@ -64,13 +73,8 @@ Sync11Service.prototype = { storageURL: null, metaURL: null, cryptoKeyURL: null, - // The cluster URL comes via the ClusterManager object, which in the FxA - // world is ebbedded in the token returned from the token server. - _clusterURL: null, - get serverURL() { - return Svc.Prefs.get("serverURL"); - }, + get serverURL() Svc.Prefs.get("serverURL"), set serverURL(value) { if (!value.endsWith("/")) { value += "/"; @@ -80,20 +84,14 @@ Sync11Service.prototype = { if (value == this.serverURL) return; + // A new server most likely uses a different cluster, so clear that Svc.Prefs.set("serverURL", value); - - // A new server most likely uses a different cluster, so clear that. - this._clusterURL = null; + Svc.Prefs.reset("clusterURL"); }, - get clusterURL() { - return this._clusterURL || ""; - }, + get clusterURL() Svc.Prefs.get("clusterURL", ""), set clusterURL(value) { - if (value != null && typeof value != "string") { - throw new Error("cluster must be a string, got " + (typeof value)); - } - this._clusterURL = value; + Svc.Prefs.set("clusterURL", value); this._updateCachedURLs(); }, @@ -171,16 +169,8 @@ Sync11Service.prototype = { _updateCachedURLs: function _updateCachedURLs() { // Nothing to cache yet if we don't have the building blocks - if (!this.clusterURL || !this.identity.username) { - // Also reset all other URLs used by Sync to ensure we aren't accidentally - // using one cached earlier - if there's no cluster URL any cached ones - // are invalid. - this.infoURL = undefined; - this.storageURL = undefined; - this.metaURL = undefined; - this.cryptoKeysURL = undefined; + if (!this.clusterURL || !this.identity.username) return; - } this._log.debug("Caching URLs under storage user base: " + this.userBaseURL); @@ -315,6 +305,21 @@ Sync11Service.prototype = { return false; }, + // The global "enabled" state comes from prefs, and will be set to false + // whenever the UI that exposes what to sync finds all Sync engines disabled. + get enabled() { + return Svc.Prefs.get("enabled"); + }, + set enabled(val) { + // There's no real reason to impose this other than to catch someone doing + // something we don't expect with bad consequences - all setting of this + // pref are in the UI code and external to this module. + if (val) { + throw new Error("Only disabling via this setter is supported"); + } + Svc.Prefs.set("enabled", val); + }, + /** * Prepare to initialize the rest of Weave after waiting a little bit */ @@ -344,8 +349,6 @@ Sync11Service.prototype = { this._clusterManager = this.identity.createClusterManager(this); this.recordManager = new RecordManager(this); - this.enabled = true; - this._registerEngines(); let ua = Cc["@mozilla.org/network/protocol;1?name=http"]. @@ -359,7 +362,6 @@ Sync11Service.prototype = { } Svc.Obs.add("weave:service:setup-complete", this); - Svc.Obs.add("sync:collection_changed", this); // Pulled from FxAccountsCommon Svc.Prefs.observe("engine.", this); this.scheduler = new SyncScheduler(this); @@ -375,6 +377,12 @@ Sync11Service.prototype = { Svc.Obs.notify("weave:engine:start-tracking"); } + // Telemetry probes to indicate if the user is using custom servers. + for (let [probeName, prefName] of Iterator(TELEMETRY_CUSTOM_SERVER_PREFS)) { + let isCustomized = Services.prefs.prefHasUserValue(prefName); + Services.telemetry.getHistogramById(probeName).add(isCustomized); + } + // Send an event now that Weave service is ready. We don't do this // synchronously so that observers can import this module before // registering an observer. @@ -424,7 +432,7 @@ Sync11Service.prototype = { // Map each old pref to the current pref branch let oldPref = new Preferences(oldPrefBranch); - for (let pref of oldPrefNames) + for each (let pref in oldPrefNames) Svc.Prefs.set(pref, oldPref.get(pref)); // Remove all the old prefs and remember that we've migrated @@ -472,7 +480,8 @@ Sync11Service.prototype = { this.engineManager.register(ns[engineName]); } catch (ex) { - this._log.warn("Could not register engine " + name, ex); + this._log.warn("Could not register engine " + name + ": " + + CommonUtils.exceptionStr(ex)); } } @@ -486,13 +495,6 @@ Sync11Service.prototype = { observe: function observe(subject, topic, data) { switch (topic) { - // Ideally this observer should be in the SyncScheduler, but it would require - // some work to know about the sync specific engines. We should move this there once it does. - case "sync:collection_changed": - if (data.includes("clients")) { - this.sync([]); // [] = clients collection only - } - break; case "weave:service:setup-complete": let status = this._checkSetup(); if (status != STATUS_DISABLED && status != CLIENT_NOT_CONFIGURED) @@ -557,8 +559,7 @@ Sync11Service.prototype = { // Always check for errors; this is also where we look for X-Weave-Alert. this.errorHandler.checkServerError(info); if (!info.success) { - this._log.error("Aborting sync: failed to get collections.") - throw info; + throw "Aborting sync: failed to get collections."; } return info; }, @@ -675,13 +676,21 @@ Sync11Service.prototype = { } catch (ex) { // This means no keys are present, or there's a network error. - this._log.debug("Failed to fetch and verify keys", ex); + this._log.debug("Failed to fetch and verify keys: " + + Utils.exceptionStr(ex)); this.errorHandler.checkServerError(ex); return false; } }, verifyLogin: function verifyLogin(allow40XRecovery = true) { + // If the identity isn't ready it might not know the username... + if (!this.identity.readyToAuthenticate) { + this._log.info("Not ready to authenticate in verifyLogin."); + this.status.login = LOGIN_FAILED_NOT_READY; + return false; + } + if (!this.identity.username) { this._log.warn("No username in verifyLogin."); this.status.login = LOGIN_FAILED_NO_USERNAME; @@ -753,12 +762,8 @@ Sync11Service.prototype = { return this.verifyLogin(false); } - // We must have the right cluster, but the server doesn't expect us. - // The implications of this depend on the identity being used - for - // the legacy identity, it's an authoritatively "incorrect password", - // (ie, LOGIN_FAILED_LOGIN_REJECTED) but for FxA it probably means - // "transient error fetching auth token". - this.status.login = this.identity.loginStatusFromVerification404(); + // We must have the right cluster, but the server doesn't expect us + this.status.login = LOGIN_FAILED_LOGIN_REJECTED; return false; default: @@ -769,7 +774,7 @@ Sync11Service.prototype = { } } catch (ex) { // Must have failed on some network issue - this._log.debug("verifyLogin failed", ex); + this._log.debug("verifyLogin failed: " + Utils.exceptionStr(ex)); this.status.login = LOGIN_FAILED_NETWORK_ERROR; this.errorHandler.checkServerError(ex); return false; @@ -842,7 +847,8 @@ Sync11Service.prototype = { try { cb.wait(); } catch (ex) { - this._log.debug("Password change failed", ex); + this._log.debug("Password change failed: " + + CommonUtils.exceptionStr(ex)); return false; } @@ -884,11 +890,12 @@ Sync11Service.prototype = { // Deletion doesn't make sense if we aren't set up yet! if (this.clusterURL != "") { // Clear client-specific data from the server, including disabled engines. - for (let engine of [this.clientsEngine].concat(this.engineManager.getAll())) { + for each (let engine in [this.clientsEngine].concat(this.engineManager.getAll())) { try { engine.removeClientData(); } catch(ex) { - this._log.warn(`Deleting client data for ${engine.name} failed`, ex); + this._log.warn("Deleting client data for " + engine.name + " failed:" + + Utils.exceptionStr(ex)); } } this._log.debug("Finished deleting client data."); @@ -914,7 +921,6 @@ Sync11Service.prototype = { this._ignorePrefObserver = true; Svc.Prefs.resetBranch(""); this._ignorePrefObserver = false; - this.clusterURL = null; Svc.Prefs.set("lastversion", WEAVE_VERSION); @@ -931,22 +937,25 @@ Sync11Service.prototype = { return; } - try { - this.identity.finalize(); - // an observer so the FxA migration code can take some action before - // the new identity is created. - Svc.Obs.notify("weave:service:start-over:init-identity"); - this.identity.username = ""; - this.status.__authManager = null; - this.identity = Status._authManager; - this._clusterManager = this.identity.createClusterManager(this); - Svc.Obs.notify("weave:service:start-over:finish"); - } catch (err) { - this._log.error("startOver failed to re-initialize the identity manager: " + err); - // Still send the observer notification so the current state is - // reflected in the UI. - Svc.Obs.notify("weave:service:start-over:finish"); - } + this.identity.finalize().then( + () => { + // an observer so the FxA migration code can take some action before + // the new identity is created. + Svc.Obs.notify("weave:service:start-over:init-identity"); + this.identity.username = ""; + this.status.__authManager = null; + this.identity = Status._authManager; + this._clusterManager = this.identity.createClusterManager(this); + Svc.Obs.notify("weave:service:start-over:finish"); + } + ).then(null, + err => { + this._log.error("startOver failed to re-initialize the identity manager: " + err); + // Still send the observer notification so the current state is + // reflected in the UI. + Svc.Obs.notify("weave:service:start-over:finish"); + } + ); }, persistLogin: function persistLogin() { @@ -981,12 +990,8 @@ Sync11Service.prototype = { } // Ask the identity manager to explicitly login now. - this._log.info("Logging in the user."); let cb = Async.makeSpinningCallback(); - this.identity.ensureLoggedIn().then( - () => cb(null), - err => cb(err || "ensureLoggedIn failed") - ); + this.identity.ensureLoggedIn().then(cb, cb); // Just let any errors bubble up - they've more context than we do! cb.wait(); @@ -997,9 +1002,9 @@ Sync11Service.prototype = { && (username || password || passphrase)) { Svc.Obs.notify("weave:service:setup-complete"); } + this._log.info("Logging in the user."); this._updateCachedURLs(); - this._log.info("User logged in successfully - verifying login."); if (!this.verifyLogin()) { // verifyLogin sets the failure states here. throw "Login failed: " + this.status.login; @@ -1064,49 +1069,11 @@ Sync11Service.prototype = { } }, - // Note: returns false if we failed for a reason other than the server not yet - // supporting the api. - _fetchServerConfiguration() { - if (Svc.Prefs.get("APILevel") >= 2) { - // This is similar to _fetchInfo, but with different error handling. - // Only supported by later sync implementations. - - let infoURL = this.userBaseURL + "info/configuration"; - this._log.debug("Fetching server configuration", infoURL); - let configResponse; - try { - configResponse = this.resource(infoURL).get(); - } catch (ex) { - // This is probably a network or similar error. - this._log.warn("Failed to fetch info/configuration", ex); - this.errorHandler.checkServerError(ex); - return false; - } - - if (configResponse.status == 404) { - // This server doesn't support the URL yet - that's OK. - this._log.debug("info/configuration returned 404 - using default upload semantics"); - } else if (configResponse.status != 200) { - this._log.warn(`info/configuration returned ${configResponse.status} - using default configuration`); - this.errorHandler.checkServerError(configResponse); - return false; - } else { - this.serverConfiguration = configResponse.obj; - } - this._log.trace("info/configuration for this server", this.serverConfiguration); - } - return true; - }, - // Stuff we need to do after login, before we can really do // anything (e.g. key setup). _remoteSetup: function _remoteSetup(infoResponse) { let reset = false; - if (!this._fetchServerConfiguration()) { - return false; - } - this._log.debug("Fetching global metadata record"); let meta = this.recordManager.get(this.metaURL); @@ -1133,7 +1100,7 @@ Sync11Service.prototype = { return false; } - if (this.recordManager.response.status == 404) { + if (!this.recordManager.response.success || !newMeta) { this._log.debug("No meta/global record on the server. Creating one."); newMeta = new WBORecord("meta", "global"); newMeta.payload.syncID = this.syncID; @@ -1143,16 +1110,10 @@ Sync11Service.prototype = { newMeta.isNew = true; this.recordManager.set(this.metaURL, newMeta); - let uploadRes = newMeta.upload(this.resource(this.metaURL)); - if (!uploadRes.success) { + if (!newMeta.upload(this.resource(this.metaURL)).success) { this._log.warn("Unable to upload new meta/global. Failing remote setup."); - this.errorHandler.checkServerError(uploadRes); return false; } - } else if (!newMeta) { - this._log.warn("Unable to get meta/global. Failing remote setup."); - this.errorHandler.checkServerError(this.recordManager.response); - return false; } else { // If newMeta, then it stands to reason that meta != null. newMeta.isNew = meta.isNew; @@ -1293,9 +1254,13 @@ Sync11Service.prototype = { return reason; }, - sync: function sync(engineNamesToSync) { - let dateStr = Utils.formatTimestamp(new Date()); - this._log.debug("User-Agent: " + Utils.userAgent); + sync: function sync() { + if (!this.enabled) { + this._log.debug("Not syncing as Sync is disabled."); + return; + } + let dateStr = new Date().toLocaleFormat(LOG_DATE_FORMAT); + this._log.debug("User-Agent: " + SyncStorageRequest.prototype.userAgent); this._log.info("Starting sync at " + dateStr); this._catch(function () { // Make sure we're logged in. @@ -1309,14 +1274,14 @@ Sync11Service.prototype = { else { this._log.trace("In sync: no need to login."); } - return this._lockedSync(engineNamesToSync); + return this._lockedSync.apply(this, arguments); })(); }, /** * Sync up engines with the server. */ - _lockedSync: function _lockedSync(engineNamesToSync) { + _lockedSync: function _lockedSync() { return this._lock("service.js: sync", this._notify("sync", "", function onNotify() { @@ -1327,7 +1292,7 @@ Sync11Service.prototype = { let cb = Async.makeSpinningCallback(); synchronizer.onComplete = cb; - synchronizer.sync(engineNamesToSync); + synchronizer.sync(); // wait() throws if the first argument is truthy, which is exactly what // we want. let result = cb.wait(); @@ -1338,31 +1303,27 @@ Sync11Service.prototype = { // We successfully synchronized. // Check if the identity wants to pre-fetch a migration sentinel from // the server. - // Only supported by Sync server API level 2+ // If we have no clusterURL, we are probably doing a node reassignment // so don't attempt to get it in that case. - if (Svc.Prefs.get("APILevel") >= 2 && this.clusterURL) { - this.identity.prefetchMigrationSentinel(this); + //if (this.clusterURL) { + // this.identity.prefetchMigrationSentinel(this); + //} + + // Now let's update our declined engines. + let meta = this.recordManager.get(this.metaURL); + if (!meta) { + this._log.warn("No meta/global; can't update declined state."); + return; } - // Now let's update our declined engines (but only if we have a metaURL; - // if Sync failed due to no node we will not have one) - if (this.metaURL) { - let meta = this.recordManager.get(this.metaURL); - if (!meta) { - this._log.warn("No meta/global; can't update declined state."); - return; - } - - let declinedEngines = new DeclinedEngines(this); - let didChange = declinedEngines.updateDeclined(meta, this.engineManager); - if (!didChange) { - this._log.info("No change to declined engines. Not reuploading meta/global."); - return; - } - - this.uploadMetaGlobal(meta); + let declinedEngines = new DeclinedEngines(this); + let didChange = declinedEngines.updateDeclined(meta, this.engineManager); + if (!didChange) { + this._log.info("No change to declined engines. Not reuploading meta/global."); + return; } + + this.uploadMetaGlobal(meta); }))(); }, @@ -1536,7 +1497,7 @@ Sync11Service.prototype = { // Wipe everything we know about except meta because we just uploaded it let engines = [this.clientsEngine].concat(this.engineManager.getAll()); - let collections = engines.map(engine => engine.name); + let collections = [engine.name for each (engine in engines)]; // TODO: there's a bug here. We should be calling resetClient, no? // Generate, upload, and download new keys. Do this last so we don't wipe @@ -1555,7 +1516,6 @@ Sync11Service.prototype = { */ wipeServer: function wipeServer(collections) { let response; - let histogram = Services.telemetry.getHistogramById("WEAVE_WIPE_SERVER_SUCCEEDED"); if (!collections) { // Strip the trailing slash. let res = this.resource(this.storageURL.slice(0, -1)); @@ -1563,17 +1523,14 @@ Sync11Service.prototype = { try { response = res.delete(); } catch (ex) { - this._log.debug("Failed to wipe server", ex); - histogram.add(false); + this._log.debug("Failed to wipe server: " + CommonUtils.exceptionStr(ex)); throw ex; } if (response.status != 200 && response.status != 404) { this._log.debug("Aborting wipeServer. Server responded with " + response.status + " response for " + this.storageURL); - histogram.add(false); throw response; } - histogram.add(true); return response.headers["x-weave-timestamp"]; } @@ -1583,15 +1540,14 @@ Sync11Service.prototype = { try { response = this.resource(url).delete(); } catch (ex) { - this._log.debug("Failed to wipe '" + name + "' collection", ex); - histogram.add(false); + this._log.debug("Failed to wipe '" + name + "' collection: " + + Utils.exceptionStr(ex)); throw ex; } if (response.status != 200 && response.status != 404) { this._log.debug("Aborting wipeServer. Server responded with " + response.status + " response for " + url); - histogram.add(false); throw response; } @@ -1599,7 +1555,7 @@ Sync11Service.prototype = { timestamp = response.headers["x-weave-timestamp"]; } } - histogram.add(true); + return timestamp; }, @@ -1623,7 +1579,7 @@ Sync11Service.prototype = { } // Fully wipe each engine if it's able to decrypt data - for (let engine of engines) { + for each (let engine in engines) { if (engine.canDecrypt()) { engine.wipeClient(); } @@ -1701,7 +1657,7 @@ Sync11Service.prototype = { } // Have each engine drop any temporary meta data - for (let engine of engines) { + for each (let engine in engines) { engine.resetClient(); } })(); @@ -1731,7 +1687,8 @@ Sync11Service.prototype = { return this.getStorageRequest(url).get(function onComplete(error) { // Note: 'this' is the request. if (error) { - this._log.debug("Failed to retrieve '" + info_type + "'", error); + this._log.debug("Failed to retrieve '" + info_type + "': " + + Utils.exceptionStr(error)); return callback(error); } if (this.response.status != 200) { diff --git a/services/sync/modules/stages/cluster.js b/services/sync/modules/stages/cluster.js index 7665ce825..dd358bf98 100644 --- a/services/sync/modules/stages/cluster.js +++ b/services/sync/modules/stages/cluster.js @@ -4,7 +4,7 @@ this.EXPORTED_SYMBOLS = ["ClusterManager"]; -var {utils: Cu} = Components; +const {utils: Cu} = Components; Cu.import("resource://gre/modules/Log.jsm"); Cu.import("resource://services-sync/constants.js"); @@ -80,9 +80,6 @@ ClusterManager.prototype = { return false; } - // Convert from the funky "String object with additional properties" that - // resource.js returns to a plain-old string. - cluster = cluster.toString(); // Don't update stuff if we already have the right cluster if (cluster == this.service.clusterURL) { return false; @@ -90,6 +87,7 @@ ClusterManager.prototype = { this._log.debug("Setting cluster to " + cluster); this.service.clusterURL = cluster; + Svc.Prefs.set("lastClusterUpdate", Date.now().toString()); return true; }, diff --git a/services/sync/modules/stages/declined.js b/services/sync/modules/stages/declined.js index ff8a14181..b0877e929 100644 --- a/services/sync/modules/stages/declined.js +++ b/services/sync/modules/stages/declined.js @@ -11,7 +11,7 @@ this.EXPORTED_SYMBOLS = ["DeclinedEngines"]; -var {utils: Cu} = Components; +const {utils: Cu} = Components; Cu.import("resource://services-sync/constants.js"); Cu.import("resource://gre/modules/Log.jsm"); @@ -29,8 +29,8 @@ this.DeclinedEngines = function (service) { } this.DeclinedEngines.prototype = { updateDeclined: function (meta, engineManager=this.service.engineManager) { - let enabled = new Set(engineManager.getEnabled().map(e => e.name)); - let known = new Set(engineManager.getAll().map(e => e.name)); + let enabled = new Set([e.name for each (e in engineManager.getEnabled())]); + let known = new Set([e.name for each (e in engineManager.getAll())]); let remoteDeclined = new Set(meta.payload.declined || []); let localDeclined = new Set(engineManager.getDeclined()); diff --git a/services/sync/modules/stages/enginesync.js b/services/sync/modules/stages/enginesync.js index a00a2f48b..ed91adddb 100644 --- a/services/sync/modules/stages/enginesync.js +++ b/services/sync/modules/stages/enginesync.js @@ -8,16 +8,13 @@ this.EXPORTED_SYMBOLS = ["EngineSynchronizer"]; -var {utils: Cu} = Components; +const {utils: Cu} = Components; Cu.import("resource://gre/modules/Log.jsm"); Cu.import("resource://services-sync/constants.js"); Cu.import("resource://services-sync/engines.js"); Cu.import("resource://services-sync/policies.js"); Cu.import("resource://services-sync/util.js"); -Cu.import("resource://services-common/observers.js"); -Cu.import("resource://services-common/async.js"); -Cu.import("resource://gre/modules/Task.jsm"); /** * Perform synchronization of engines. @@ -34,7 +31,7 @@ this.EngineSynchronizer = function EngineSynchronizer(service) { } EngineSynchronizer.prototype = { - sync: function sync(engineNamesToSync) { + sync: function sync() { if (!this.onComplete) { throw new Error("onComplete handler not installed."); } @@ -99,9 +96,6 @@ EngineSynchronizer.prototype = { return; } - // We only honor the "hint" of what engines to Sync if this isn't - // a first sync. - let allowEnginesHint = false; // Wipe data in the desired direction if necessary switch (Svc.Prefs.get("firstSync")) { case "resetClient": @@ -113,9 +107,6 @@ EngineSynchronizer.prototype = { case "wipeRemote": this.service.wipeRemote(engineManager.enabledEngineNames); break; - default: - allowEnginesHint = true; - break; } if (this.service.clientsEngine.localCommands) { @@ -145,31 +136,20 @@ EngineSynchronizer.prototype = { try { this._updateEnabledEngines(); } catch (ex) { - this._log.debug("Updating enabled engines failed", ex); + this._log.debug("Updating enabled engines failed: " + + Utils.exceptionStr(ex)); this.service.errorHandler.checkServerError(ex); this.onComplete(ex); return; } - // If the engines to sync has been specified, we sync in the order specified. - let enginesToSync; - if (allowEnginesHint && engineNamesToSync) { - this._log.info("Syncing specified engines", engineNamesToSync); - enginesToSync = engineManager.get(engineNamesToSync).filter(e => e.enabled); - } else { - this._log.info("Syncing all enabled engines."); - enginesToSync = engineManager.getEnabled(); - } try { - // We don't bother validating engines that failed to sync. - let enginesToValidate = []; - for (let engine of enginesToSync) { + for (let engine of engineManager.getEnabled()) { // If there's any problems with syncing the engine, report the failure if (!(this._syncEngine(engine)) || this.service.status.enforceBackoff) { this._log.info("Aborting sync for failure in " + engine.name); break; } - enginesToValidate.push(engine); } // If _syncEngine fails for a 401, we might not have a cluster URL here. @@ -195,8 +175,6 @@ EngineSynchronizer.prototype = { } } - Async.promiseSpinningly(this._tryValidateEngines(enginesToValidate)); - // If there were no sync engine failures if (this.service.status.service != SYNC_FAILED_PARTIAL) { Svc.Prefs.set("lastSync", new Date().toString()); @@ -206,7 +184,7 @@ EngineSynchronizer.prototype = { Svc.Prefs.reset("firstSync"); let syncTime = ((Date.now() - startTime) / 1000).toFixed(2); - let dateStr = Utils.formatTimestamp(new Date()); + let dateStr = new Date().toLocaleFormat(LOG_DATE_FORMAT); this._log.info("Sync completed at " + dateStr + " after " + syncTime + " secs."); } @@ -214,106 +192,6 @@ EngineSynchronizer.prototype = { this.onComplete(null); }, - _tryValidateEngines: Task.async(function* (recentlySyncedEngines) { - if (!Services.telemetry.canRecordBase || !Svc.Prefs.get("validation.enabled", false)) { - this._log.info("Skipping validation: validation or telemetry reporting is disabled"); - return; - } - - let lastValidation = Svc.Prefs.get("validation.lastTime", 0); - let validationInterval = Svc.Prefs.get("validation.interval"); - let nowSeconds = Math.floor(Date.now() / 1000); - - if (nowSeconds - lastValidation < validationInterval) { - this._log.info("Skipping validation: too recent since last validation attempt"); - return; - } - // Update the time now, even if we may return false still. We don't want to - // check the rest of these more frequently than once a day. - Svc.Prefs.set("validation.lastTime", nowSeconds); - - // Validation only occurs a certain percentage of the time. - let validationProbability = Svc.Prefs.get("validation.percentageChance", 0) / 100.0; - if (validationProbability < Math.random()) { - this._log.info("Skipping validation: Probability threshold not met"); - return; - } - let maxRecords = Svc.Prefs.get("validation.maxRecords"); - if (!maxRecords) { - // Don't bother asking the server for the counts if we know validation - // won't happen anyway. - return; - } - - // maxRecords of -1 means "any number", so we can skip asking the server. - // Used for tests. - let info; - if (maxRecords < 0) { - info = {}; - for (let e of recentlySyncedEngines) { - info[e.name] = 1; // needs to be < maxRecords - } - maxRecords = 2; - } else { - - let collectionCountsURL = this.service.userBaseURL + "info/collection_counts"; - try { - let infoResp = this.service._fetchInfo(collectionCountsURL); - if (!infoResp.success) { - this._log.error("Can't run validation: request to info/collection_counts responded with " - + resp.status); - return; - } - info = infoResp.obj; // might throw because obj is a getter which parses json. - } catch (e) { - // Not running validation is totally fine, so we just write an error log and return. - this._log.error("Can't run validation: Caught error when fetching counts", e); - return; - } - } - - if (!info) { - return; - } - - let engineLookup = new Map(recentlySyncedEngines.map(e => [e.name, e])); - let toRun = []; - for (let [engineName, recordCount] of Object.entries(info)) { - let engine = engineLookup.get(engineName); - if (recordCount > maxRecords || !engine) { - this._log.debug(`Skipping validation for ${engineName} because it's not an engine or ` + - `the number of records (${recordCount}) is greater than the maximum allowed (${maxRecords}).`); - continue; - } - let validator = engine.getValidator(); - if (!validator) { - continue; - } - // Put this in an array so that we know how many we're going to do, so we - // don't tell users we're going to run some validators when we aren't. - toRun.push({ engine, validator }); - } - - if (!toRun.length) { - return; - } - Services.console.logStringMessage( - "Sync is about to run a consistency check. This may be slow, and " + - "can be controlled using the pref \"services.sync.validation.enabled\".\n" + - "If you encounter any problems because of this, please file a bug."); - for (let { validator, engine } of toRun) { - try { - let result = yield validator.validate(engine); - Observers.notify("weave:engine:validate:finish", result, engine.name); - } catch (e) { - this._log.error(`Failed to run validation on ${engine.name}!`, e); - Observers.notify("weave:engine:validate:error", e, engine.name) - // Keep validating -- there's no reason to think that a failure for one - // validator would mean the others will fail. - } - } - }), - // Returns true if sync should proceed. // false / no return value means sync should be aborted. _syncEngine: function _syncEngine(engine) { @@ -346,15 +224,8 @@ EngineSynchronizer.prototype = { // If we're the only client, and no engines are marked as enabled, // thumb our noses at the server data: it can't be right. // Belt-and-suspenders approach to Bug 615926. - let hasEnabledEngines = false; - for (let e in meta.payload.engines) { - if (e != "clients") { - hasEnabledEngines = true; - break; - } - } - - if ((numClients <= 1) && !hasEnabledEngines) { + if ((numClients <= 1) && + ([e for (e in meta.payload.engines) if (e != "clients")].length == 0)) { this._log.info("One client and no enabled engines: not touching local engine status."); return; } @@ -418,7 +289,7 @@ EngineSynchronizer.prototype = { } // Any remaining engines were either enabled locally or disabled remotely. - for (let engineName of enabled) { + for each (let engineName in enabled) { let engine = engineManager.get(engineName); if (Svc.Prefs.get("engineStatusChanged." + engine.prefName, false)) { this._log.trace("The " + engineName + " engine was enabled locally."); diff --git a/services/sync/modules/status.js b/services/sync/modules/status.js index 100bc7965..19dff9712 100644 --- a/services/sync/modules/status.js +++ b/services/sync/modules/status.js @@ -4,10 +4,10 @@ this.EXPORTED_SYMBOLS = ["Status"]; -var Cc = Components.classes; -var Ci = Components.interfaces; -var Cr = Components.results; -var Cu = Components.utils; +const Cc = Components.classes; +const Ci = Components.interfaces; +const Cr = Components.results; +const Cu = Components.utils; Cu.import("resource://services-sync/constants.js"); Cu.import("resource://gre/modules/Log.jsm"); @@ -30,7 +30,10 @@ this.Status = { .wrappedJSObject; let idClass = service.fxAccountsEnabled ? BrowserIDManager : IdentityManager; this.__authManager = new idClass(); - this.__authManager.initialize(); + // .initialize returns a promise, so we need to spin until it resolves. + let cb = Async.makeSpinningCallback(); + this.__authManager.initialize().then(cb, cb); + cb.wait(); return this.__authManager; }, diff --git a/services/sync/modules/telemetry.js b/services/sync/modules/telemetry.js deleted file mode 100644 index c311387f7..000000000 --- a/services/sync/modules/telemetry.js +++ /dev/null @@ -1,578 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this file, - * You can obtain one at http://mozilla.org/MPL/2.0/. */ - -"use strict"; - -const {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components; - -this.EXPORTED_SYMBOLS = ["SyncTelemetry"]; - -Cu.import("resource://services-sync/browserid_identity.js"); -Cu.import("resource://services-sync/main.js"); -Cu.import("resource://services-sync/status.js"); -Cu.import("resource://services-sync/util.js"); -Cu.import("resource://services-common/observers.js"); -Cu.import("resource://services-common/async.js"); -Cu.import("resource://gre/modules/Log.jsm"); -Cu.import("resource://gre/modules/TelemetryController.jsm"); -Cu.import("resource://gre/modules/FxAccounts.jsm"); -Cu.import("resource://gre/modules/XPCOMUtils.jsm"); -Cu.import("resource://gre/modules/osfile.jsm", this); - -let constants = {}; -Cu.import("resource://services-sync/constants.js", constants); - -var fxAccountsCommon = {}; -Cu.import("resource://gre/modules/FxAccountsCommon.js", fxAccountsCommon); - -XPCOMUtils.defineLazyServiceGetter(this, "Telemetry", - "@mozilla.org/base/telemetry;1", - "nsITelemetry"); - -const log = Log.repository.getLogger("Sync.Telemetry"); - -const TOPICS = [ - "profile-before-change", - "weave:service:sync:start", - "weave:service:sync:finish", - "weave:service:sync:error", - - "weave:engine:sync:start", - "weave:engine:sync:finish", - "weave:engine:sync:error", - "weave:engine:sync:applied", - "weave:engine:sync:uploaded", - "weave:engine:validate:finish", - "weave:engine:validate:error", -]; - -const PING_FORMAT_VERSION = 1; - -// The set of engines we record telemetry for - any other engines are ignored. -const ENGINES = new Set(["addons", "bookmarks", "clients", "forms", "history", - "passwords", "prefs", "tabs", "extension-storage"]); - -// A regex we can use to replace the profile dir in error messages. We use a -// regexp so we can simply replace all case-insensitive occurences. -// This escaping function is from: -// https://developer.mozilla.org/en/docs/Web/JavaScript/Guide/Regular_Expressions -const reProfileDir = new RegExp( - OS.Constants.Path.profileDir.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"), - "gi"); - -function transformError(error, engineName) { - if (Async.isShutdownException(error)) { - return { name: "shutdownerror" }; - } - - if (typeof error === "string") { - if (error.startsWith("error.")) { - // This is hacky, but I can't imagine that it's not also accurate. - return { name: "othererror", error }; - } - // There's a chance the profiledir is in the error string which is PII we - // want to avoid including in the ping. - error = error.replace(reProfileDir, "[profileDir]"); - return { name: "unexpectederror", error }; - } - - if (error.failureCode) { - return { name: "othererror", error: error.failureCode }; - } - - if (error instanceof AuthenticationError) { - return { name: "autherror", from: error.source }; - } - - if (error instanceof Ci.mozIStorageError) { - return { name: "sqlerror", code: error.result }; - } - - let httpCode = error.status || - (error.response && error.response.status) || - error.code; - - if (httpCode) { - return { name: "httperror", code: httpCode }; - } - - if (error.result) { - return { name: "nserror", code: error.result }; - } - - return { - name: "unexpectederror", - // as above, remove the profile dir value. - error: String(error).replace(reProfileDir, "[profileDir]") - } -} - -function tryGetMonotonicTimestamp() { - try { - return Telemetry.msSinceProcessStart(); - } catch (e) { - log.warn("Unable to get a monotonic timestamp!"); - return -1; - } -} - -function timeDeltaFrom(monotonicStartTime) { - let now = tryGetMonotonicTimestamp(); - if (monotonicStartTime !== -1 && now !== -1) { - return Math.round(now - monotonicStartTime); - } - return -1; -} - -class EngineRecord { - constructor(name) { - // startTime is in ms from process start, but is monotonic (unlike Date.now()) - // so we need to keep both it and when. - this.startTime = tryGetMonotonicTimestamp(); - this.name = name; - } - - toJSON() { - let result = Object.assign({}, this); - delete result.startTime; - return result; - } - - finished(error) { - let took = timeDeltaFrom(this.startTime); - if (took > 0) { - this.took = took; - } - if (error) { - this.failureReason = transformError(error, this.name); - } - } - - recordApplied(counts) { - if (this.incoming) { - log.error(`Incoming records applied multiple times for engine ${this.name}!`); - return; - } - if (this.name === "clients" && !counts.failed) { - // ignore successful application of client records - // since otherwise they show up every time and are meaningless. - return; - } - - let incomingData = {}; - let properties = ["applied", "failed", "newFailed", "reconciled"]; - // Only record non-zero properties and only record incoming at all if - // there's at least one property we care about. - for (let property of properties) { - if (counts[property]) { - incomingData[property] = counts[property]; - this.incoming = incomingData; - } - } - } - - recordValidation(validationResult) { - if (this.validation) { - log.error(`Multiple validations occurred for engine ${this.name}!`); - return; - } - let { problems, version, duration, recordCount } = validationResult; - let validation = { - version: version || 0, - checked: recordCount || 0, - }; - if (duration > 0) { - validation.took = Math.round(duration); - } - let summarized = problems.getSummary(true).filter(({count}) => count > 0); - if (summarized.length) { - validation.problems = summarized; - } - this.validation = validation; - } - - recordValidationError(e) { - if (this.validation) { - log.error(`Multiple validations occurred for engine ${this.name}!`); - return; - } - - this.validation = { - failureReason: transformError(e) - }; - } - - recordUploaded(counts) { - if (counts.sent || counts.failed) { - if (!this.outgoing) { - this.outgoing = []; - } - this.outgoing.push({ - sent: counts.sent || undefined, - failed: counts.failed || undefined, - }); - } - } -} - -class TelemetryRecord { - constructor(allowedEngines) { - this.allowedEngines = allowedEngines; - // Our failure reason. This property only exists in the generated ping if an - // error actually occurred. - this.failureReason = undefined; - this.uid = ""; - this.when = Date.now(); - this.startTime = tryGetMonotonicTimestamp(); - this.took = 0; // will be set later. - - // All engines that have finished (ie, does not include the "current" one) - // We omit this from the ping if it's empty. - this.engines = []; - // The engine that has started but not yet stopped. - this.currentEngine = null; - } - - toJSON() { - let result = { - when: this.when, - uid: this.uid, - took: this.took, - failureReason: this.failureReason, - status: this.status, - deviceID: this.deviceID, - devices: this.devices, - }; - let engines = []; - for (let engine of this.engines) { - engines.push(engine.toJSON()); - } - if (engines.length > 0) { - result.engines = engines; - } - return result; - } - - finished(error) { - this.took = timeDeltaFrom(this.startTime); - if (this.currentEngine != null) { - log.error("Finished called for the sync before the current engine finished"); - this.currentEngine.finished(null); - this.onEngineStop(this.currentEngine.name); - } - if (error) { - this.failureReason = transformError(error); - } - - // We don't bother including the "devices" field if we can't come up with a - // UID or device ID for *this* device -- If that's the case, any data we'd - // put there would be likely to be full of garbage anyway. - let includeDeviceInfo = false; - try { - this.uid = Weave.Service.identity.hashedUID(); - let deviceID = Weave.Service.identity.deviceID(); - if (deviceID) { - // Combine the raw device id with the metrics uid to create a stable - // unique identifier that can't be mapped back to the user's FxA - // identity without knowing the metrics HMAC key. - this.deviceID = Utils.sha256(deviceID + this.uid); - includeDeviceInfo = true; - } - } catch (e) { - this.uid = "0".repeat(32); - this.deviceID = undefined; - } - - if (includeDeviceInfo) { - let remoteDevices = Weave.Service.clientsEngine.remoteClients; - this.devices = remoteDevices.map(device => { - return { - os: device.os, - version: device.version, - id: Utils.sha256(device.id + this.uid) - }; - }); - } - - // Check for engine statuses. -- We do this now, and not in engine.finished - // to make sure any statuses that get set "late" are recorded - for (let engine of this.engines) { - let status = Status.engines[engine.name]; - if (status && status !== constants.ENGINE_SUCCEEDED) { - engine.status = status; - } - } - - let statusObject = {}; - - let serviceStatus = Status.service; - if (serviceStatus && serviceStatus !== constants.STATUS_OK) { - statusObject.service = serviceStatus; - this.status = statusObject; - } - let syncStatus = Status.sync; - if (syncStatus && syncStatus !== constants.SYNC_SUCCEEDED) { - statusObject.sync = syncStatus; - this.status = statusObject; - } - } - - onEngineStart(engineName) { - if (this._shouldIgnoreEngine(engineName, false)) { - return; - } - - if (this.currentEngine) { - log.error(`Being told that engine ${engineName} has started, but current engine ${ - this.currentEngine.name} hasn't stopped`); - // Just discard the current engine rather than making up data for it. - } - this.currentEngine = new EngineRecord(engineName); - } - - onEngineStop(engineName, error) { - // We only care if it's the current engine if we have a current engine. - if (this._shouldIgnoreEngine(engineName, !!this.currentEngine)) { - return; - } - if (!this.currentEngine) { - // It's possible for us to get an error before the start message of an engine - // (somehow), in which case we still want to record that error. - if (!error) { - return; - } - log.error(`Error triggered on ${engineName} when no current engine exists: ${error}`); - this.currentEngine = new EngineRecord(engineName); - } - this.currentEngine.finished(error); - this.engines.push(this.currentEngine); - this.currentEngine = null; - } - - onEngineApplied(engineName, counts) { - if (this._shouldIgnoreEngine(engineName)) { - return; - } - this.currentEngine.recordApplied(counts); - } - - onEngineValidated(engineName, validationData) { - if (this._shouldIgnoreEngine(engineName, false)) { - return; - } - let engine = this.engines.find(e => e.name === engineName); - if (!engine && this.currentEngine && engineName === this.currentEngine.name) { - engine = this.currentEngine; - } - if (engine) { - engine.recordValidation(validationData); - } else { - log.warn(`Validation event triggered for engine ${engineName}, which hasn't been synced!`); - } - } - - onEngineValidateError(engineName, error) { - if (this._shouldIgnoreEngine(engineName, false)) { - return; - } - let engine = this.engines.find(e => e.name === engineName); - if (!engine && this.currentEngine && engineName === this.currentEngine.name) { - engine = this.currentEngine; - } - if (engine) { - engine.recordValidationError(error); - } else { - log.warn(`Validation failure event triggered for engine ${engineName}, which hasn't been synced!`); - } - } - - onEngineUploaded(engineName, counts) { - if (this._shouldIgnoreEngine(engineName)) { - return; - } - this.currentEngine.recordUploaded(counts); - } - - _shouldIgnoreEngine(engineName, shouldBeCurrent = true) { - if (!this.allowedEngines.has(engineName)) { - log.info(`Notification for engine ${engineName}, but we aren't recording telemetry for it`); - return true; - } - if (shouldBeCurrent) { - if (!this.currentEngine || engineName != this.currentEngine.name) { - log.error(`Notification for engine ${engineName} but it isn't current`); - return true; - } - } - return false; - } -} - -class SyncTelemetryImpl { - constructor(allowedEngines) { - log.level = Log.Level[Svc.Prefs.get("log.logger.telemetry", "Trace")]; - // This is accessible so we can enable custom engines during tests. - this.allowedEngines = allowedEngines; - this.current = null; - this.setupObservers(); - - this.payloads = []; - this.discarded = 0; - this.maxPayloadCount = Svc.Prefs.get("telemetry.maxPayloadCount"); - this.submissionInterval = Svc.Prefs.get("telemetry.submissionInterval") * 1000; - this.lastSubmissionTime = Telemetry.msSinceProcessStart(); - } - - getPingJSON(reason) { - return { - why: reason, - discarded: this.discarded || undefined, - version: PING_FORMAT_VERSION, - syncs: this.payloads.slice(), - }; - } - - finish(reason) { - // Note that we might be in the middle of a sync right now, and so we don't - // want to touch this.current. - let result = this.getPingJSON(reason); - this.payloads = []; - this.discarded = 0; - this.submit(result); - } - - setupObservers() { - for (let topic of TOPICS) { - Observers.add(topic, this, this); - } - } - - shutdown() { - this.finish("shutdown"); - for (let topic of TOPICS) { - Observers.remove(topic, this, this); - } - } - - submit(record) { - // We still call submit() with possibly illegal payloads so that tests can - // know that the ping was built. We don't end up submitting them, however. - if (record.syncs.length) { - log.trace(`submitting ${record.syncs.length} sync record(s) to telemetry`); - TelemetryController.submitExternalPing("sync", record); - } - } - - - onSyncStarted() { - if (this.current) { - log.warn("Observed weave:service:sync:start, but we're already recording a sync!"); - // Just discard the old record, consistent with our handling of engines, above. - this.current = null; - } - this.current = new TelemetryRecord(this.allowedEngines); - } - - _checkCurrent(topic) { - if (!this.current) { - log.warn(`Observed notification ${topic} but no current sync is being recorded.`); - return false; - } - return true; - } - - onSyncFinished(error) { - if (!this.current) { - log.warn("onSyncFinished but we aren't recording"); - return; - } - this.current.finished(error); - if (this.payloads.length < this.maxPayloadCount) { - this.payloads.push(this.current.toJSON()); - } else { - ++this.discarded; - } - this.current = null; - if ((Telemetry.msSinceProcessStart() - this.lastSubmissionTime) > this.submissionInterval) { - this.finish("schedule"); - this.lastSubmissionTime = Telemetry.msSinceProcessStart(); - } - } - - observe(subject, topic, data) { - log.trace(`observed ${topic} ${data}`); - - switch (topic) { - case "profile-before-change": - this.shutdown(); - break; - - /* sync itself state changes */ - case "weave:service:sync:start": - this.onSyncStarted(); - break; - - case "weave:service:sync:finish": - if (this._checkCurrent(topic)) { - this.onSyncFinished(null); - } - break; - - case "weave:service:sync:error": - // argument needs to be truthy (this should always be the case) - this.onSyncFinished(subject || "Unknown"); - break; - - /* engine sync state changes */ - case "weave:engine:sync:start": - if (this._checkCurrent(topic)) { - this.current.onEngineStart(data); - } - break; - case "weave:engine:sync:finish": - if (this._checkCurrent(topic)) { - this.current.onEngineStop(data, null); - } - break; - - case "weave:engine:sync:error": - if (this._checkCurrent(topic)) { - // argument needs to be truthy (this should always be the case) - this.current.onEngineStop(data, subject || "Unknown"); - } - break; - - /* engine counts */ - case "weave:engine:sync:applied": - if (this._checkCurrent(topic)) { - this.current.onEngineApplied(data, subject); - } - break; - - case "weave:engine:sync:uploaded": - if (this._checkCurrent(topic)) { - this.current.onEngineUploaded(data, subject); - } - break; - - case "weave:engine:validate:finish": - if (this._checkCurrent(topic)) { - this.current.onEngineValidated(data, subject); - } - break; - - case "weave:engine:validate:error": - if (this._checkCurrent(topic)) { - this.current.onEngineValidateError(data, subject || "Unknown"); - } - break; - - default: - log.warn(`unexpected observer topic ${topic}`); - break; - } - } -} - -this.SyncTelemetry = new SyncTelemetryImpl(ENGINES); diff --git a/services/sync/modules/userapi.js b/services/sync/modules/userapi.js index e906440bd..ec77d63e2 100644 --- a/services/sync/modules/userapi.js +++ b/services/sync/modules/userapi.js @@ -8,7 +8,7 @@ this.EXPORTED_SYMBOLS = [ "UserAPI10Client", ]; -var {utils: Cu} = Components; +const {utils: Cu} = Components; Cu.import("resource://gre/modules/Log.jsm"); Cu.import("resource://services-common/rest.js"); diff --git a/services/sync/modules/util.js b/services/sync/modules/util.js index e9dbcb37d..67cc3f063 100644 --- a/services/sync/modules/util.js +++ b/services/sync/modules/util.js @@ -4,7 +4,7 @@ this.EXPORTED_SYMBOLS = ["XPCOMUtils", "Services", "Utils", "Async", "Svc", "Str"]; -var {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components; +const {classes: Cc, interfaces: Ci, results: Cr, utils: Cu} = Components; Cu.import("resource://gre/modules/Log.jsm"); Cu.import("resource://services-common/observers.js"); @@ -35,6 +35,8 @@ this.Utils = { // In the ideal world, references to these would be removed. nextTick: CommonUtils.nextTick, namedTimer: CommonUtils.namedTimer, + exceptionStr: CommonUtils.exceptionStr, + stackTrace: CommonUtils.stackTrace, makeURI: CommonUtils.makeURI, encodeUTF8: CommonUtils.encodeUTF8, decodeUTF8: CommonUtils.decodeUTF8, @@ -52,7 +54,6 @@ this.Utils = { digestBytes: CryptoUtils.digestBytes, sha1: CryptoUtils.sha1, sha1Base32: CryptoUtils.sha1Base32, - sha256: CryptoUtils.sha256, makeHMACKey: CryptoUtils.makeHMACKey, makeHMACHasher: CryptoUtils.makeHMACHasher, hkdfExpand: CryptoUtils.hkdfExpand, @@ -61,25 +62,6 @@ this.Utils = { getHTTPMACSHA1Header: CryptoUtils.getHTTPMACSHA1Header, /** - * The string to use as the base User-Agent in Sync requests. - * This string will look something like - * - * Firefox/49.0a1 (Windows NT 6.1; WOW64; rv:46.0) FxSync/1.51.0.20160516142357.desktop - */ - _userAgent: null, - get userAgent() { - if (!this._userAgent) { - let hph = Cc["@mozilla.org/network/protocol;1?name=http"].getService(Ci.nsIHttpProtocolHandler); - this._userAgent = - Services.appinfo.name + "/" + Services.appinfo.version + // Product. - " (" + hph.oscpu + ")" + // (oscpu) - " FxSync/" + WEAVE_VERSION + "." + // Sync. - Services.appinfo.appBuildID + "."; // Build. - } - return this._userAgent + Svc.Prefs.get("client.type", "desktop"); - }, - - /** * Wrap a function to catch all exceptions and log them * * @usage MyObj._catch = Utils.catch; @@ -95,7 +77,7 @@ this.Utils = { return func.call(thisArg); } catch(ex) { - thisArg._log.debug("Exception calling " + (func.name || "anonymous function"), ex); + thisArg._log.debug("Exception: " + Utils.exceptionStr(ex)); if (exceptionCallback) { return exceptionCallback.call(thisArg, ex); } @@ -271,14 +253,14 @@ this.Utils = { */ base32ToFriendly: function base32ToFriendly(input) { return input.toLowerCase() - .replace(/l/g, '8') - .replace(/o/g, '9'); + .replace("l", '8', "g") + .replace("o", '9', "g"); }, base32FromFriendly: function base32FromFriendly(input) { return input.toUpperCase() - .replace(/8/g, 'L') - .replace(/9/g, 'O'); + .replace("8", 'L', "g") + .replace("9", 'O', "g"); }, /** @@ -356,14 +338,12 @@ this.Utils = { try { json = yield CommonUtils.readJSON(path); + } catch (e if e instanceof OS.File.Error && e.becauseNoSuchFile) { + // Ignore non-existent files. } catch (e) { - if (e instanceof OS.File.Error && e.becauseNoSuchFile) { - // Ignore non-existent files, but explicitly return null. - json = null; - } else { - if (that._log) { - that._log.debug("Failed to load json", e); - } + if (that._log) { + that._log.debug("Failed to load json: " + + CommonUtils.exceptionStr(e)); } } @@ -411,52 +391,6 @@ this.Utils = { } }), - /** - * Move a json file in the profile directory. Will fail if a file exists at the - * destination. - * - * @returns a promise that resolves to undefined on success, or rejects on failure - * - * @param aFrom - * Current path to the JSON file saved on disk, relative to profileDir/weave - * .json will be appended to the file name. - * @param aTo - * New path to the JSON file saved on disk, relative to profileDir/weave - * .json will be appended to the file name. - * @param that - * Object to use for logging - */ - jsonMove(aFrom, aTo, that) { - let pathFrom = OS.Path.join(OS.Constants.Path.profileDir, "weave", - ...(aFrom + ".json").split("/")); - let pathTo = OS.Path.join(OS.Constants.Path.profileDir, "weave", - ...(aTo + ".json").split("/")); - if (that._log) { - that._log.trace("Moving " + pathFrom + " to " + pathTo); - } - return OS.File.move(pathFrom, pathTo, { noOverwrite: true }); - }, - - /** - * Removes a json file in the profile directory. - * - * @returns a promise that resolves to undefined on success, or rejects on failure - * - * @param filePath - * Current path to the JSON file saved on disk, relative to profileDir/weave - * .json will be appended to the file name. - * @param that - * Object to use for logging - */ - jsonRemove(filePath, that) { - let path = OS.Path.join(OS.Constants.Path.profileDir, "weave", - ...(filePath + ".json").split("/")); - if (that._log) { - that._log.trace("Deleting " + path); - } - return OS.File.remove(path, { ignoreAbsent: true }); - }, - getErrorString: function Utils_getErrorString(error, args) { try { return Str.errors.get(error, args || null); @@ -543,7 +477,7 @@ this.Utils = { // 20-char sync key. if (pp.length == 23 && - [5, 11, 17].every(i => pp[i] == '-')) { + [5, 11, 17].every(function(i) pp[i] == '-')) { return pp.slice(0, 5) + pp.slice(6, 11) + pp.slice(12, 17) + pp.slice(18, 23); @@ -551,7 +485,7 @@ this.Utils = { // "Modern" 26-char key. if (pp.length == 31 && - [1, 7, 13, 19, 25].every(i => pp[i] == '-')) { + [1, 7, 13, 19, 25].every(function(i) pp[i] == '-')) { return pp.slice(0, 1) + pp.slice(2, 7) + pp.slice(8, 13) + pp.slice(14, 19) @@ -681,12 +615,30 @@ this.Utils = { * Get the FxA identity hosts. */ getSyncCredentialsHostsFxA: function() { + // This is somewhat expensive and the result static, so we cache the result. + if (this._syncCredentialsHostsFxA) { + return this._syncCredentialsHostsFxA; + } let result = new Set(); // the FxA host result.add(FxAccountsCommon.FXA_PWDMGR_HOST); - // We used to include the FxA hosts (hence the Set() result) but we now - // don't give them special treatment (hence the Set() with exactly 1 item) - return result; + // + // The FxA hosts - these almost certainly all have the same hostname, but + // better safe than sorry... + for (let prefName of ["identity.fxaccounts.remote.force_auth.uri", + "identity.fxaccounts.remote.signup.uri", + "identity.fxaccounts.remote.signin.uri", + "identity.fxaccounts.settings.uri"]) { + let prefVal; + try { + prefVal = Services.prefs.getCharPref(prefName); + } catch (_) { + continue; + } + let uri = Services.io.newURI(prefVal, null, null); + result.add(uri.prePath); + } + return this._syncCredentialsHostsFxA = result; }, getDefaultDeviceName() { @@ -720,32 +672,6 @@ this.Utils = { Cc["@mozilla.org/network/protocol;1?name=http"].getService(Ci.nsIHttpProtocolHandler).oscpu; return Str.sync.get("client.name2", [user, appName, system]); - }, - - getDeviceName() { - const deviceName = Svc.Prefs.get("client.name", ""); - - if (deviceName === "") { - return this.getDefaultDeviceName(); - } - - return deviceName; - }, - - getDeviceType() { - return Svc.Prefs.get("client.type", DEVICE_TYPE_DESKTOP); - }, - - formatTimestamp(date) { - // Format timestamp as: "%Y-%m-%d %H:%M:%S" - let year = String(date.getFullYear()); - let month = String(date.getMonth() + 1).padStart(2, "0"); - let day = String(date.getDate()).padStart(2, "0"); - let hours = String(date.getHours()).padStart(2, "0"); - let minutes = String(date.getMinutes()).padStart(2, "0"); - let seconds = String(date.getSeconds()).padStart(2, "0"); - - return `${year}-${month}-${day} ${hours}:${minutes}:${seconds}`; } }; @@ -764,7 +690,7 @@ Svc.Prefs = new Preferences(PREFS_BRANCH); Svc.DefaultPrefs = new Preferences({branch: PREFS_BRANCH, defaultBranch: true}); Svc.Obs = Observers; -var _sessionCID = Services.appinfo.ID == SEAMONKEY_ID ? +let _sessionCID = Services.appinfo.ID == SEAMONKEY_ID ? "@mozilla.org/suite/sessionstore;1" : "@mozilla.org/browser/sessionstore;1"; |