Import Tycho weave client

1 files changed, 120 insertions, 163 deletions

diff --git a/services/sync/modules/service.js b/services/sync/modules/service.js
index 32e047f53..637760b8f 100644
--- a/services/sync/modules/service.js
+++ b/services/sync/modules/service.js
@@ -4,10 +4,10 @@
 
 this.EXPORTED_SYMBOLS = ["Service"];
 
-var Cc = Components.classes;
-var Ci = Components.interfaces;
-var Cr = Components.results;
-var Cu = Components.utils;
+const Cc = Components.classes;
+const Ci = Components.interfaces;
+const Cr = Components.results;
+const Cu = Components.utils;
 
 // How long before refreshing the cluster
 const CLUSTER_BACKOFF = 5 * 60 * 1000; // 5 minutes
@@ -21,6 +21,7 @@ const KEYS_WBO = "keys";
 Cu.import("resource://gre/modules/Preferences.jsm");
 Cu.import("resource://gre/modules/XPCOMUtils.jsm");
 Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://services-common/utils.js");
 Cu.import("resource://services-sync/constants.js");
 Cu.import("resource://services-sync/engines.js");
 Cu.import("resource://services-sync/engines/clients.js");
@@ -32,7 +33,6 @@ Cu.import("resource://services-sync/rest.js");
 Cu.import("resource://services-sync/stages/enginesync.js");
 Cu.import("resource://services-sync/stages/declined.js");
 Cu.import("resource://services-sync/status.js");
-Cu.import("resource://services-sync/telemetry.js");
 Cu.import("resource://services-sync/userapi.js");
 Cu.import("resource://services-sync/util.js");
 
@@ -51,6 +51,15 @@ const STORAGE_INFO_TYPES = [INFO_COLLECTIONS,
                             INFO_COLLECTION_COUNTS,
                             INFO_QUOTA];
 
+// A structure mapping a (boolean) telemetry probe name to a preference name.
+// The probe will record true if the pref is modified, false otherwise.
+const TELEMETRY_CUSTOM_SERVER_PREFS = {
+  WEAVE_CUSTOM_LEGACY_SERVER_CONFIGURATION: "services.sync.serverURL",
+  WEAVE_CUSTOM_FXA_SERVER_CONFIGURATION: "identity.fxaccounts.auth.uri",
+  WEAVE_CUSTOM_TOKEN_SERVER_CONFIGURATION: "services.sync.tokenServerURI",
+};
+
+
 function Sync11Service() {
   this._notify = Utils.notify("weave:service:");
 }
@@ -64,13 +73,8 @@ Sync11Service.prototype = {
   storageURL: null,
   metaURL: null,
   cryptoKeyURL: null,
-  // The cluster URL comes via the ClusterManager object, which in the FxA
-  // world is ebbedded in the token returned from the token server.
-  _clusterURL: null,
 
-  get serverURL() {
-    return Svc.Prefs.get("serverURL");
-  },
+  get serverURL() Svc.Prefs.get("serverURL"),
   set serverURL(value) {
     if (!value.endsWith("/")) {
       value += "/";
@@ -80,20 +84,14 @@ Sync11Service.prototype = {
     if (value == this.serverURL)
       return;
 
+    // A new server most likely uses a different cluster, so clear that
     Svc.Prefs.set("serverURL", value);
-
-    // A new server most likely uses a different cluster, so clear that.
-    this._clusterURL = null;
+    Svc.Prefs.reset("clusterURL");
   },
 
-  get clusterURL() {
-    return this._clusterURL || "";
-  },
+  get clusterURL() Svc.Prefs.get("clusterURL", ""),
   set clusterURL(value) {
-    if (value != null && typeof value != "string") {
-      throw new Error("cluster must be a string, got " + (typeof value));
-    }
-    this._clusterURL = value;
+    Svc.Prefs.set("clusterURL", value);
     this._updateCachedURLs();
   },
 
@@ -171,16 +169,8 @@ Sync11Service.prototype = {
 
   _updateCachedURLs: function _updateCachedURLs() {
     // Nothing to cache yet if we don't have the building blocks
-    if (!this.clusterURL || !this.identity.username) {
-      // Also reset all other URLs used by Sync to ensure we aren't accidentally
-      // using one cached earlier - if there's no cluster URL any cached ones
-      // are invalid.
-      this.infoURL = undefined;
-      this.storageURL = undefined;
-      this.metaURL = undefined;
-      this.cryptoKeysURL = undefined;
+    if (!this.clusterURL || !this.identity.username)
       return;
-    }
 
     this._log.debug("Caching URLs under storage user base: " + this.userBaseURL);
 
@@ -315,6 +305,21 @@ Sync11Service.prototype = {
     return false;
   },
 
+  // The global "enabled" state comes from prefs, and will be set to false
+  // whenever the UI that exposes what to sync finds all Sync engines disabled.
+  get enabled() {
+    return Svc.Prefs.get("enabled");
+  },
+  set enabled(val) {
+    // There's no real reason to impose this other than to catch someone doing
+    // something we don't expect with bad consequences - all setting of this
+    // pref are in the UI code and external to this module.
+    if (val) {
+      throw new Error("Only disabling via this setter is supported");
+    }
+    Svc.Prefs.set("enabled", val);
+  },
+
   /**
    * Prepare to initialize the rest of Weave after waiting a little bit
    */
@@ -344,8 +349,6 @@ Sync11Service.prototype = {
     this._clusterManager = this.identity.createClusterManager(this);
     this.recordManager = new RecordManager(this);
 
-    this.enabled = true;
-
     this._registerEngines();
 
     let ua = Cc["@mozilla.org/network/protocol;1?name=http"].
@@ -359,7 +362,6 @@ Sync11Service.prototype = {
     }
 
     Svc.Obs.add("weave:service:setup-complete", this);
-    Svc.Obs.add("sync:collection_changed", this); // Pulled from FxAccountsCommon
     Svc.Prefs.observe("engine.", this);
 
     this.scheduler = new SyncScheduler(this);
@@ -375,6 +377,12 @@ Sync11Service.prototype = {
       Svc.Obs.notify("weave:engine:start-tracking");
     }
 
+    // Telemetry probes to indicate if the user is using custom servers.
+    for (let [probeName, prefName] of Iterator(TELEMETRY_CUSTOM_SERVER_PREFS)) {
+      let isCustomized = Services.prefs.prefHasUserValue(prefName);
+      Services.telemetry.getHistogramById(probeName).add(isCustomized);
+    }
+
     // Send an event now that Weave service is ready.  We don't do this
     // synchronously so that observers can import this module before
     // registering an observer.
@@ -424,7 +432,7 @@ Sync11Service.prototype = {
 
     // Map each old pref to the current pref branch
     let oldPref = new Preferences(oldPrefBranch);
-    for (let pref of oldPrefNames)
+    for each (let pref in oldPrefNames)
       Svc.Prefs.set(pref, oldPref.get(pref));
 
     // Remove all the old prefs and remember that we've migrated
@@ -472,7 +480,8 @@ Sync11Service.prototype = {
 
         this.engineManager.register(ns[engineName]);
       } catch (ex) {
-        this._log.warn("Could not register engine " + name, ex);
+        this._log.warn("Could not register engine " + name + ": " +
+                       CommonUtils.exceptionStr(ex));
       }
     }
 
@@ -486,13 +495,6 @@ Sync11Service.prototype = {
 
   observe: function observe(subject, topic, data) {
     switch (topic) {
-      // Ideally this observer should be in the SyncScheduler, but it would require
-      // some work to know about the sync specific engines. We should move this there once it does.
-      case "sync:collection_changed":
-        if (data.includes("clients")) {
-          this.sync([]); // [] = clients collection only
-        }
-        break;
       case "weave:service:setup-complete":
         let status = this._checkSetup();
         if (status != STATUS_DISABLED && status != CLIENT_NOT_CONFIGURED)
@@ -557,8 +559,7 @@ Sync11Service.prototype = {
     // Always check for errors; this is also where we look for X-Weave-Alert.
     this.errorHandler.checkServerError(info);
     if (!info.success) {
-      this._log.error("Aborting sync: failed to get collections.")
-      throw info;
+      throw "Aborting sync: failed to get collections.";
     }
     return info;
   },
@@ -675,13 +676,21 @@ Sync11Service.prototype = {
 
     } catch (ex) {
       // This means no keys are present, or there's a network error.
-      this._log.debug("Failed to fetch and verify keys", ex);
+      this._log.debug("Failed to fetch and verify keys: "
+                      + Utils.exceptionStr(ex));
       this.errorHandler.checkServerError(ex);
       return false;
     }
   },
 
   verifyLogin: function verifyLogin(allow40XRecovery = true) {
+    // If the identity isn't ready it  might not know the username...
+    if (!this.identity.readyToAuthenticate) {
+      this._log.info("Not ready to authenticate in verifyLogin.");
+      this.status.login = LOGIN_FAILED_NOT_READY;
+      return false;
+    }
+
     if (!this.identity.username) {
       this._log.warn("No username in verifyLogin.");
       this.status.login = LOGIN_FAILED_NO_USERNAME;
@@ -753,12 +762,8 @@ Sync11Service.prototype = {
             return this.verifyLogin(false);
           }
 
-          // We must have the right cluster, but the server doesn't expect us.
-          // The implications of this depend on the identity being used - for
-          // the legacy identity, it's an authoritatively "incorrect password",
-          // (ie, LOGIN_FAILED_LOGIN_REJECTED) but for FxA it probably means
-          // "transient error fetching auth token".
-          this.status.login = this.identity.loginStatusFromVerification404();
+          // We must have the right cluster, but the server doesn't expect us
+          this.status.login = LOGIN_FAILED_LOGIN_REJECTED;
           return false;
 
         default:
@@ -769,7 +774,7 @@ Sync11Service.prototype = {
       }
     } catch (ex) {
       // Must have failed on some network issue
-      this._log.debug("verifyLogin failed", ex);
+      this._log.debug("verifyLogin failed: " + Utils.exceptionStr(ex));
       this.status.login = LOGIN_FAILED_NETWORK_ERROR;
       this.errorHandler.checkServerError(ex);
       return false;
@@ -842,7 +847,8 @@ Sync11Service.prototype = {
     try {
       cb.wait();
     } catch (ex) {
-      this._log.debug("Password change failed", ex);
+      this._log.debug("Password change failed: " +
+                      CommonUtils.exceptionStr(ex));
       return false;
     }
 
@@ -884,11 +890,12 @@ Sync11Service.prototype = {
     // Deletion doesn't make sense if we aren't set up yet!
     if (this.clusterURL != "") {
       // Clear client-specific data from the server, including disabled engines.
-      for (let engine of [this.clientsEngine].concat(this.engineManager.getAll())) {
+      for each (let engine in [this.clientsEngine].concat(this.engineManager.getAll())) {
         try {
           engine.removeClientData();
         } catch(ex) {
-          this._log.warn(`Deleting client data for ${engine.name} failed`, ex);
+          this._log.warn("Deleting client data for " + engine.name + " failed:"
+                         + Utils.exceptionStr(ex));
         }
       }
       this._log.debug("Finished deleting client data.");
@@ -914,7 +921,6 @@ Sync11Service.prototype = {
     this._ignorePrefObserver = true;
     Svc.Prefs.resetBranch("");
     this._ignorePrefObserver = false;
-    this.clusterURL = null;
 
     Svc.Prefs.set("lastversion", WEAVE_VERSION);
 
@@ -931,22 +937,25 @@ Sync11Service.prototype = {
       return;
     }
 
-    try {
-      this.identity.finalize();
-      // an observer so the FxA migration code can take some action before
-      // the new identity is created.
-      Svc.Obs.notify("weave:service:start-over:init-identity");
-      this.identity.username = "";
-      this.status.__authManager = null;
-      this.identity = Status._authManager;
-      this._clusterManager = this.identity.createClusterManager(this);
-      Svc.Obs.notify("weave:service:start-over:finish");
-    } catch (err) {
-      this._log.error("startOver failed to re-initialize the identity manager: " + err);
-      // Still send the observer notification so the current state is
-      // reflected in the UI.
-      Svc.Obs.notify("weave:service:start-over:finish");
-    }
+    this.identity.finalize().then(
+      () => {
+        // an observer so the FxA migration code can take some action before
+        // the new identity is created.
+        Svc.Obs.notify("weave:service:start-over:init-identity");
+        this.identity.username = "";
+        this.status.__authManager = null;
+        this.identity = Status._authManager;
+        this._clusterManager = this.identity.createClusterManager(this);
+        Svc.Obs.notify("weave:service:start-over:finish");
+      }
+    ).then(null,
+      err => {
+        this._log.error("startOver failed to re-initialize the identity manager: " + err);
+        // Still send the observer notification so the current state is
+        // reflected in the UI.
+        Svc.Obs.notify("weave:service:start-over:finish");
+      }
+    );
   },
 
   persistLogin: function persistLogin() {
@@ -981,12 +990,8 @@ Sync11Service.prototype = {
       }
 
       // Ask the identity manager to explicitly login now.
-      this._log.info("Logging in the user.");
       let cb = Async.makeSpinningCallback();
-      this.identity.ensureLoggedIn().then(
-        () => cb(null),
-        err => cb(err || "ensureLoggedIn failed")
-      );
+      this.identity.ensureLoggedIn().then(cb, cb);
 
       // Just let any errors bubble up - they've more context than we do!
       cb.wait();
@@ -997,9 +1002,9 @@ Sync11Service.prototype = {
           && (username || password || passphrase)) {
         Svc.Obs.notify("weave:service:setup-complete");
       }
+      this._log.info("Logging in the user.");
       this._updateCachedURLs();
 
-      this._log.info("User logged in successfully - verifying login.");
       if (!this.verifyLogin()) {
         // verifyLogin sets the failure states here.
         throw "Login failed: " + this.status.login;
@@ -1064,49 +1069,11 @@ Sync11Service.prototype = {
     }
   },
 
-  // Note: returns false if we failed for a reason other than the server not yet
-  // supporting the api.
-  _fetchServerConfiguration() {
-    if (Svc.Prefs.get("APILevel") >= 2) {
-      // This is similar to _fetchInfo, but with different error handling.
-      // Only supported by later sync implementations.
-
-      let infoURL = this.userBaseURL + "info/configuration";
-      this._log.debug("Fetching server configuration", infoURL);
-      let configResponse;
-      try {
-        configResponse = this.resource(infoURL).get();
-      } catch (ex) {
-        // This is probably a network or similar error.
-        this._log.warn("Failed to fetch info/configuration", ex);
-        this.errorHandler.checkServerError(ex);
-        return false;
-      }
-
-      if (configResponse.status == 404) {
-        // This server doesn't support the URL yet - that's OK.
-        this._log.debug("info/configuration returned 404 - using default upload semantics");
-      } else if (configResponse.status != 200) {
-        this._log.warn(`info/configuration returned ${configResponse.status} - using default configuration`);
-        this.errorHandler.checkServerError(configResponse);
-        return false;
-      } else {
-        this.serverConfiguration = configResponse.obj;
-      }
-      this._log.trace("info/configuration for this server", this.serverConfiguration);
-    }
-    return true;
-  },
-
   // Stuff we need to do after login, before we can really do
   // anything (e.g. key setup).
   _remoteSetup: function _remoteSetup(infoResponse) {
     let reset = false;
 
-    if (!this._fetchServerConfiguration()) {
-      return false;
-    }
-
     this._log.debug("Fetching global metadata record");
     let meta = this.recordManager.get(this.metaURL);
 
@@ -1133,7 +1100,7 @@ Sync11Service.prototype = {
         return false;
       }
 
-      if (this.recordManager.response.status == 404) {
+      if (!this.recordManager.response.success || !newMeta) {
         this._log.debug("No meta/global record on the server. Creating one.");
         newMeta = new WBORecord("meta", "global");
         newMeta.payload.syncID = this.syncID;
@@ -1143,16 +1110,10 @@ Sync11Service.prototype = {
         newMeta.isNew = true;
 
         this.recordManager.set(this.metaURL, newMeta);
-        let uploadRes = newMeta.upload(this.resource(this.metaURL));
-        if (!uploadRes.success) {
+        if (!newMeta.upload(this.resource(this.metaURL)).success) {
           this._log.warn("Unable to upload new meta/global. Failing remote setup.");
-          this.errorHandler.checkServerError(uploadRes);
           return false;
         }
-      } else if (!newMeta) {
-        this._log.warn("Unable to get meta/global. Failing remote setup.");
-        this.errorHandler.checkServerError(this.recordManager.response);
-        return false;
       } else {
         // If newMeta, then it stands to reason that meta != null.
         newMeta.isNew   = meta.isNew;
@@ -1293,9 +1254,13 @@ Sync11Service.prototype = {
     return reason;
   },
 
-  sync: function sync(engineNamesToSync) {
-    let dateStr = Utils.formatTimestamp(new Date());
-    this._log.debug("User-Agent: " + Utils.userAgent);
+  sync: function sync() {
+    if (!this.enabled) {
+      this._log.debug("Not syncing as Sync is disabled.");
+      return;
+    }
+    let dateStr = new Date().toLocaleFormat(LOG_DATE_FORMAT);
+    this._log.debug("User-Agent: " + SyncStorageRequest.prototype.userAgent);
     this._log.info("Starting sync at " + dateStr);
     this._catch(function () {
       // Make sure we're logged in.
@@ -1309,14 +1274,14 @@ Sync11Service.prototype = {
       else {
         this._log.trace("In sync: no need to login.");
       }
-      return this._lockedSync(engineNamesToSync);
+      return this._lockedSync.apply(this, arguments);
     })();
   },
 
   /**
    * Sync up engines with the server.
    */
-  _lockedSync: function _lockedSync(engineNamesToSync) {
+  _lockedSync: function _lockedSync() {
     return this._lock("service.js: sync",
                       this._notify("sync", "", function onNotify() {
 
@@ -1327,7 +1292,7 @@ Sync11Service.prototype = {
       let cb = Async.makeSpinningCallback();
       synchronizer.onComplete = cb;
 
-      synchronizer.sync(engineNamesToSync);
+      synchronizer.sync();
       // wait() throws if the first argument is truthy, which is exactly what
       // we want.
       let result = cb.wait();
@@ -1338,31 +1303,27 @@ Sync11Service.prototype = {
       // We successfully synchronized.
       // Check if the identity wants to pre-fetch a migration sentinel from
       // the server.
-      // Only supported by Sync server API level 2+
       // If we have no clusterURL, we are probably doing a node reassignment
       // so don't attempt to get it in that case.
-      if (Svc.Prefs.get("APILevel") >= 2 && this.clusterURL) {
-        this.identity.prefetchMigrationSentinel(this);
+      //if (this.clusterURL) {
+      //  this.identity.prefetchMigrationSentinel(this);
+      //}
+
+      // Now let's update our declined engines.
+      let meta = this.recordManager.get(this.metaURL);
+      if (!meta) {
+        this._log.warn("No meta/global; can't update declined state.");
+        return;
       }
 
-      // Now let's update our declined engines (but only if we have a metaURL;
-      // if Sync failed due to no node we will not have one)
-      if (this.metaURL) {
-        let meta = this.recordManager.get(this.metaURL);
-        if (!meta) {
-          this._log.warn("No meta/global; can't update declined state.");
-          return;
-        }
-
-        let declinedEngines = new DeclinedEngines(this);
-        let didChange = declinedEngines.updateDeclined(meta, this.engineManager);
-        if (!didChange) {
-          this._log.info("No change to declined engines. Not reuploading meta/global.");
-          return;
-        }
-
-        this.uploadMetaGlobal(meta);
+      let declinedEngines = new DeclinedEngines(this);
+      let didChange = declinedEngines.updateDeclined(meta, this.engineManager);
+      if (!didChange) {
+        this._log.info("No change to declined engines. Not reuploading meta/global.");
+        return;
       }
+
+      this.uploadMetaGlobal(meta);
     }))();
   },
 
@@ -1536,7 +1497,7 @@ Sync11Service.prototype = {
 
     // Wipe everything we know about except meta because we just uploaded it
     let engines = [this.clientsEngine].concat(this.engineManager.getAll());
-    let collections = engines.map(engine => engine.name);
+    let collections = [engine.name for each (engine in engines)];
     // TODO: there's a bug here. We should be calling resetClient, no?
 
     // Generate, upload, and download new keys. Do this last so we don't wipe
@@ -1555,7 +1516,6 @@ Sync11Service.prototype = {
    */
   wipeServer: function wipeServer(collections) {
     let response;
-    let histogram = Services.telemetry.getHistogramById("WEAVE_WIPE_SERVER_SUCCEEDED");
     if (!collections) {
       // Strip the trailing slash.
       let res = this.resource(this.storageURL.slice(0, -1));
@@ -1563,17 +1523,14 @@ Sync11Service.prototype = {
       try {
         response = res.delete();
       } catch (ex) {
-        this._log.debug("Failed to wipe server", ex);
-        histogram.add(false);
+        this._log.debug("Failed to wipe server: " + CommonUtils.exceptionStr(ex));
         throw ex;
       }
       if (response.status != 200 && response.status != 404) {
         this._log.debug("Aborting wipeServer. Server responded with " +
                         response.status + " response for " + this.storageURL);
-        histogram.add(false);
         throw response;
       }
-      histogram.add(true);
       return response.headers["x-weave-timestamp"];
     }
 
@@ -1583,15 +1540,14 @@ Sync11Service.prototype = {
       try {
         response = this.resource(url).delete();
       } catch (ex) {
-        this._log.debug("Failed to wipe '" + name + "' collection", ex);
-        histogram.add(false);
+        this._log.debug("Failed to wipe '" + name + "' collection: " +
+                        Utils.exceptionStr(ex));
         throw ex;
       }
 
       if (response.status != 200 && response.status != 404) {
         this._log.debug("Aborting wipeServer. Server responded with " +
                         response.status + " response for " + url);
-        histogram.add(false);
         throw response;
       }
 
@@ -1599,7 +1555,7 @@ Sync11Service.prototype = {
         timestamp = response.headers["x-weave-timestamp"];
       }
     }
-    histogram.add(true);
+
     return timestamp;
   },
 
@@ -1623,7 +1579,7 @@ Sync11Service.prototype = {
     }
 
     // Fully wipe each engine if it's able to decrypt data
-    for (let engine of engines) {
+    for each (let engine in engines) {
       if (engine.canDecrypt()) {
         engine.wipeClient();
       }
@@ -1701,7 +1657,7 @@ Sync11Service.prototype = {
       }
 
       // Have each engine drop any temporary meta data
-      for (let engine of engines) {
+      for each (let engine in engines) {
         engine.resetClient();
       }
     })();
@@ -1731,7 +1687,8 @@ Sync11Service.prototype = {
     return this.getStorageRequest(url).get(function onComplete(error) {
       // Note: 'this' is the request.
       if (error) {
-        this._log.debug("Failed to retrieve '" + info_type + "'", error);
+        this._log.debug("Failed to retrieve '" + info_type + "': " +
+                        Utils.exceptionStr(error));
         return callback(error);
       }
       if (this.response.status != 200) {