summaryrefslogtreecommitdiffstats
path: root/services/sync/modules/record.js
diff options
context:
space:
mode:
authorMatt A. Tobin <mattatobin@localhost.localdomain>2018-02-02 04:16:08 -0500
committerMatt A. Tobin <mattatobin@localhost.localdomain>2018-02-02 04:16:08 -0500
commit5f8de423f190bbb79a62f804151bc24824fa32d8 (patch)
tree10027f336435511475e392454359edea8e25895d /services/sync/modules/record.js
parent49ee0794b5d912db1f95dce6eb52d781dc210db5 (diff)
downloadUXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.gz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.lz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.xz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.zip
Add m-esr52 at 52.6.0
Diffstat (limited to 'services/sync/modules/record.js')
-rw-r--r--services/sync/modules/record.js1039
1 files changed, 1039 insertions, 0 deletions
diff --git a/services/sync/modules/record.js b/services/sync/modules/record.js
new file mode 100644
index 000000000..02f7f281a
--- /dev/null
+++ b/services/sync/modules/record.js
@@ -0,0 +1,1039 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+this.EXPORTED_SYMBOLS = [
+ "WBORecord",
+ "RecordManager",
+ "CryptoWrapper",
+ "CollectionKeyManager",
+ "Collection",
+];
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cr = Components.results;
+var Cu = Components.utils;
+
+const CRYPTO_COLLECTION = "crypto";
+const KEYS_WBO = "keys";
+
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://services-sync/keys.js");
+Cu.import("resource://services-sync/resource.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://services-common/async.js");
+
+this.WBORecord = function WBORecord(collection, id) {
+ this.data = {};
+ this.payload = {};
+ this.collection = collection; // Optional.
+ this.id = id; // Optional.
+}
+WBORecord.prototype = {
+ _logName: "Sync.Record.WBO",
+
+ get sortindex() {
+ if (this.data.sortindex)
+ return this.data.sortindex;
+ return 0;
+ },
+
+ // Get thyself from your URI, then deserialize.
+ // Set thine 'response' field.
+ fetch: function fetch(resource) {
+ if (!resource instanceof Resource) {
+ throw new Error("First argument must be a Resource instance.");
+ }
+
+ let r = resource.get();
+ if (r.success) {
+ this.deserialize(r); // Warning! Muffles exceptions!
+ }
+ this.response = r;
+ return this;
+ },
+
+ upload: function upload(resource) {
+ if (!resource instanceof Resource) {
+ throw new Error("First argument must be a Resource instance.");
+ }
+
+ return resource.put(this);
+ },
+
+ // Take a base URI string, with trailing slash, and return the URI of this
+ // WBO based on collection and ID.
+ uri: function(base) {
+ if (this.collection && this.id) {
+ let url = Utils.makeURI(base + this.collection + "/" + this.id);
+ url.QueryInterface(Ci.nsIURL);
+ return url;
+ }
+ return null;
+ },
+
+ deserialize: function deserialize(json) {
+ this.data = json.constructor.toString() == String ? JSON.parse(json) : json;
+
+ try {
+ // The payload is likely to be JSON, but if not, keep it as a string
+ this.payload = JSON.parse(this.payload);
+ } catch(ex) {}
+ },
+
+ toJSON: function toJSON() {
+ // Copy fields from data to be stringified, making sure payload is a string
+ let obj = {};
+ for (let [key, val] of Object.entries(this.data))
+ obj[key] = key == "payload" ? JSON.stringify(val) : val;
+ if (this.ttl)
+ obj.ttl = this.ttl;
+ return obj;
+ },
+
+ toString: function toString() {
+ return "{ " +
+ "id: " + this.id + " " +
+ "index: " + this.sortindex + " " +
+ "modified: " + this.modified + " " +
+ "ttl: " + this.ttl + " " +
+ "payload: " + JSON.stringify(this.payload) +
+ " }";
+ }
+};
+
+Utils.deferGetSet(WBORecord, "data", ["id", "modified", "sortindex", "payload"]);
+
+this.CryptoWrapper = function CryptoWrapper(collection, id) {
+ this.cleartext = {};
+ WBORecord.call(this, collection, id);
+ this.ciphertext = null;
+ this.id = id;
+}
+CryptoWrapper.prototype = {
+ __proto__: WBORecord.prototype,
+ _logName: "Sync.Record.CryptoWrapper",
+
+ ciphertextHMAC: function ciphertextHMAC(keyBundle) {
+ let hasher = keyBundle.sha256HMACHasher;
+ if (!hasher) {
+ throw "Cannot compute HMAC without an HMAC key.";
+ }
+
+ return Utils.bytesAsHex(Utils.digestUTF8(this.ciphertext, hasher));
+ },
+
+ /*
+ * Don't directly use the sync key. Instead, grab a key for this
+ * collection, which is decrypted with the sync key.
+ *
+ * Cache those keys; invalidate the cache if the time on the keys collection
+ * changes, or other auth events occur.
+ *
+ * Optional key bundle overrides the collection key lookup.
+ */
+ encrypt: function encrypt(keyBundle) {
+ if (!keyBundle) {
+ throw new Error("A key bundle must be supplied to encrypt.");
+ }
+
+ this.IV = Svc.Crypto.generateRandomIV();
+ this.ciphertext = Svc.Crypto.encrypt(JSON.stringify(this.cleartext),
+ keyBundle.encryptionKeyB64, this.IV);
+ this.hmac = this.ciphertextHMAC(keyBundle);
+ this.cleartext = null;
+ },
+
+ // Optional key bundle.
+ decrypt: function decrypt(keyBundle) {
+ if (!this.ciphertext) {
+ throw "No ciphertext: nothing to decrypt?";
+ }
+
+ if (!keyBundle) {
+ throw new Error("A key bundle must be supplied to decrypt.");
+ }
+
+ // Authenticate the encrypted blob with the expected HMAC
+ let computedHMAC = this.ciphertextHMAC(keyBundle);
+
+ if (computedHMAC != this.hmac) {
+ Utils.throwHMACMismatch(this.hmac, computedHMAC);
+ }
+
+ // Handle invalid data here. Elsewhere we assume that cleartext is an object.
+ let cleartext = Svc.Crypto.decrypt(this.ciphertext,
+ keyBundle.encryptionKeyB64, this.IV);
+ let json_result = JSON.parse(cleartext);
+
+ if (json_result && (json_result instanceof Object)) {
+ this.cleartext = json_result;
+ this.ciphertext = null;
+ } else {
+ throw "Decryption failed: result is <" + json_result + ">, not an object.";
+ }
+
+ // Verify that the encrypted id matches the requested record's id.
+ if (this.cleartext.id != this.id)
+ throw "Record id mismatch: " + this.cleartext.id + " != " + this.id;
+
+ return this.cleartext;
+ },
+
+ toString: function toString() {
+ let payload = this.deleted ? "DELETED" : JSON.stringify(this.cleartext);
+
+ return "{ " +
+ "id: " + this.id + " " +
+ "index: " + this.sortindex + " " +
+ "modified: " + this.modified + " " +
+ "ttl: " + this.ttl + " " +
+ "payload: " + payload + " " +
+ "collection: " + (this.collection || "undefined") +
+ " }";
+ },
+
+ // The custom setter below masks the parent's getter, so explicitly call it :(
+ get id() {
+ return WBORecord.prototype.__lookupGetter__("id").call(this);
+ },
+
+ // Keep both plaintext and encrypted versions of the id to verify integrity
+ set id(val) {
+ WBORecord.prototype.__lookupSetter__("id").call(this, val);
+ return this.cleartext.id = val;
+ },
+};
+
+Utils.deferGetSet(CryptoWrapper, "payload", ["ciphertext", "IV", "hmac"]);
+Utils.deferGetSet(CryptoWrapper, "cleartext", "deleted");
+
+/**
+ * An interface and caching layer for records.
+ */
+this.RecordManager = function RecordManager(service) {
+ this.service = service;
+
+ this._log = Log.repository.getLogger(this._logName);
+ this._records = {};
+}
+RecordManager.prototype = {
+ _recordType: CryptoWrapper,
+ _logName: "Sync.RecordManager",
+
+ import: function RecordMgr_import(url) {
+ this._log.trace("Importing record: " + (url.spec ? url.spec : url));
+ try {
+ // Clear out the last response with empty object if GET fails
+ this.response = {};
+ this.response = this.service.resource(url).get();
+
+ // Don't parse and save the record on failure
+ if (!this.response.success)
+ return null;
+
+ let record = new this._recordType(url);
+ record.deserialize(this.response);
+
+ return this.set(url, record);
+ } catch (ex) {
+ if (Async.isShutdownException(ex)) {
+ throw ex;
+ }
+ this._log.debug("Failed to import record", ex);
+ return null;
+ }
+ },
+
+ get: function RecordMgr_get(url) {
+ // Use a url string as the key to the hash
+ let spec = url.spec ? url.spec : url;
+ if (spec in this._records)
+ return this._records[spec];
+ return this.import(url);
+ },
+
+ set: function RecordMgr_set(url, record) {
+ let spec = url.spec ? url.spec : url;
+ return this._records[spec] = record;
+ },
+
+ contains: function RecordMgr_contains(url) {
+ if ((url.spec || url) in this._records)
+ return true;
+ return false;
+ },
+
+ clearCache: function recordMgr_clearCache() {
+ this._records = {};
+ },
+
+ del: function RecordMgr_del(url) {
+ delete this._records[url];
+ }
+};
+
+/**
+ * Keeps track of mappings between collection names ('tabs') and KeyBundles.
+ *
+ * You can update this thing simply by giving it /info/collections. It'll
+ * use the last modified time to bring itself up to date.
+ */
+this.CollectionKeyManager = function CollectionKeyManager(lastModified, default_, collections) {
+ this.lastModified = lastModified || 0;
+ this._default = default_ || null;
+ this._collections = collections || {};
+
+ this._log = Log.repository.getLogger("Sync.CollectionKeyManager");
+}
+
+// TODO: persist this locally as an Identity. Bug 610913.
+// Note that the last modified time needs to be preserved.
+CollectionKeyManager.prototype = {
+
+ /**
+ * Generate a new CollectionKeyManager that has the same attributes
+ * as this one.
+ */
+ clone() {
+ const newCollections = {};
+ for (let c in this._collections) {
+ newCollections[c] = this._collections[c];
+ }
+
+ return new CollectionKeyManager(this.lastModified, this._default, newCollections);
+ },
+
+ // Return information about old vs new keys:
+ // * same: true if two collections are equal
+ // * changed: an array of collection names that changed.
+ _compareKeyBundleCollections: function _compareKeyBundleCollections(m1, m2) {
+ let changed = [];
+
+ function process(m1, m2) {
+ for (let k1 in m1) {
+ let v1 = m1[k1];
+ let v2 = m2[k1];
+ if (!(v1 && v2 && v1.equals(v2)))
+ changed.push(k1);
+ }
+ }
+
+ // Diffs both ways.
+ process(m1, m2);
+ process(m2, m1);
+
+ // Return a sorted, unique array.
+ changed.sort();
+ let last;
+ changed = changed.filter(x => (x != last) && (last = x));
+ return {same: changed.length == 0,
+ changed: changed};
+ },
+
+ get isClear() {
+ return !this._default;
+ },
+
+ clear: function clear() {
+ this._log.info("Clearing collection keys...");
+ this.lastModified = 0;
+ this._collections = {};
+ this._default = null;
+ },
+
+ keyForCollection: function(collection) {
+ if (collection && this._collections[collection])
+ return this._collections[collection];
+
+ return this._default;
+ },
+
+ /**
+ * If `collections` (an array of strings) is provided, iterate
+ * over it and generate random keys for each collection.
+ * Create a WBO for the given data.
+ */
+ _makeWBO: function(collections, defaultBundle) {
+ let wbo = new CryptoWrapper(CRYPTO_COLLECTION, KEYS_WBO);
+ let c = {};
+ for (let k in collections) {
+ c[k] = collections[k].keyPairB64;
+ }
+ wbo.cleartext = {
+ "default": defaultBundle ? defaultBundle.keyPairB64 : null,
+ "collections": c,
+ "collection": CRYPTO_COLLECTION,
+ "id": KEYS_WBO
+ };
+ return wbo;
+ },
+
+ /**
+ * Create a WBO for the current keys.
+ */
+ asWBO: function(collection, id) {
+ return this._makeWBO(this._collections, this._default);
+ },
+
+ /**
+ * Compute a new default key, and new keys for any specified collections.
+ */
+ newKeys: function(collections) {
+ let newDefaultKeyBundle = this.newDefaultKeyBundle();
+
+ let newColls = {};
+ if (collections) {
+ collections.forEach(function (c) {
+ let b = new BulkKeyBundle(c);
+ b.generateRandom();
+ newColls[c] = b;
+ });
+ }
+ return [newDefaultKeyBundle, newColls];
+ },
+
+ /**
+ * Generates new keys, but does not replace our local copy. Use this to
+ * verify an upload before storing.
+ */
+ generateNewKeysWBO: function(collections) {
+ let newDefaultKey, newColls;
+ [newDefaultKey, newColls] = this.newKeys(collections);
+
+ return this._makeWBO(newColls, newDefaultKey);
+ },
+
+ /**
+ * Create a new default key.
+ *
+ * @returns {BulkKeyBundle}
+ */
+ newDefaultKeyBundle() {
+ const key = new BulkKeyBundle(DEFAULT_KEYBUNDLE_NAME);
+ key.generateRandom();
+ return key;
+ },
+
+ /**
+ * Create a new default key and store it as this._default, since without one you cannot use setContents.
+ */
+ generateDefaultKey() {
+ this._default = this.newDefaultKeyBundle();
+ },
+
+ /**
+ * Return true if keys are already present for each of the given
+ * collections.
+ */
+ hasKeysFor(collections) {
+ // We can't use filter() here because sometimes collections is an iterator.
+ for (let collection of collections) {
+ if (!this._collections[collection]) {
+ return false;
+ }
+ }
+ return true;
+ },
+
+ /**
+ * Return a new CollectionKeyManager that has keys for each of the
+ * given collections (creating new ones for collections where we
+ * don't already have keys).
+ */
+ ensureKeysFor(collections) {
+ const newKeys = Object.assign({}, this._collections);
+ for (let c of collections) {
+ if (newKeys[c]) {
+ continue; // don't replace existing keys
+ }
+
+ const b = new BulkKeyBundle(c);
+ b.generateRandom();
+ newKeys[c] = b;
+ }
+ return new CollectionKeyManager(this.lastModified, this._default, newKeys);
+ },
+
+ // Take the fetched info/collections WBO, checking the change
+ // time of the crypto collection.
+ updateNeeded: function(info_collections) {
+
+ this._log.info("Testing for updateNeeded. Last modified: " + this.lastModified);
+
+ // No local record of modification time? Need an update.
+ if (!this.lastModified)
+ return true;
+
+ // No keys on the server? We need an update, though our
+ // update handling will be a little more drastic...
+ if (!(CRYPTO_COLLECTION in info_collections))
+ return true;
+
+ // Otherwise, we need an update if our modification time is stale.
+ return (info_collections[CRYPTO_COLLECTION] > this.lastModified);
+ },
+
+ //
+ // Set our keys and modified time to the values fetched from the server.
+ // Returns one of three values:
+ //
+ // * If the default key was modified, return true.
+ // * If the default key was not modified, but per-collection keys were,
+ // return an array of such.
+ // * Otherwise, return false -- we were up-to-date.
+ //
+ setContents: function setContents(payload, modified) {
+
+ let self = this;
+
+ this._log.info("Setting collection keys contents. Our last modified: " +
+ this.lastModified + ", input modified: " + modified + ".");
+
+ if (!payload)
+ throw "No payload in CollectionKeyManager.setContents().";
+
+ if (!payload.default) {
+ this._log.warn("No downloaded default key: this should not occur.");
+ this._log.warn("Not clearing local keys.");
+ throw "No default key in CollectionKeyManager.setContents(). Cannot proceed.";
+ }
+
+ // Process the incoming default key.
+ let b = new BulkKeyBundle(DEFAULT_KEYBUNDLE_NAME);
+ b.keyPairB64 = payload.default;
+ let newDefault = b;
+
+ // Process the incoming collections.
+ let newCollections = {};
+ if ("collections" in payload) {
+ this._log.info("Processing downloaded per-collection keys.");
+ let colls = payload.collections;
+ for (let k in colls) {
+ let v = colls[k];
+ if (v) {
+ let keyObj = new BulkKeyBundle(k);
+ keyObj.keyPairB64 = v;
+ newCollections[k] = keyObj;
+ }
+ }
+ }
+
+ // Check to see if these are already our keys.
+ let sameDefault = (this._default && this._default.equals(newDefault));
+ let collComparison = this._compareKeyBundleCollections(newCollections, this._collections);
+ let sameColls = collComparison.same;
+
+ if (sameDefault && sameColls) {
+ self._log.info("New keys are the same as our old keys!");
+ if (modified) {
+ self._log.info("Bumped local modified time.");
+ self.lastModified = modified;
+ }
+ return false;
+ }
+
+ // Make sure things are nice and tidy before we set.
+ this.clear();
+
+ this._log.info("Saving downloaded keys.");
+ this._default = newDefault;
+ this._collections = newCollections;
+
+ // Always trust the server.
+ if (modified) {
+ self._log.info("Bumping last modified to " + modified);
+ self.lastModified = modified;
+ }
+
+ return sameDefault ? collComparison.changed : true;
+ },
+
+ updateContents: function updateContents(syncKeyBundle, storage_keys) {
+ let log = this._log;
+ log.info("Updating collection keys...");
+
+ // storage_keys is a WBO, fetched from storage/crypto/keys.
+ // Its payload is the default key, and a map of collections to keys.
+ // We lazily compute the key objects from the strings we're given.
+
+ let payload;
+ try {
+ payload = storage_keys.decrypt(syncKeyBundle);
+ } catch (ex) {
+ log.warn("Got exception \"" + ex + "\" decrypting storage keys with sync key.");
+ log.info("Aborting updateContents. Rethrowing.");
+ throw ex;
+ }
+
+ let r = this.setContents(payload, storage_keys.modified);
+ log.info("Collection keys updated.");
+ return r;
+ }
+}
+
+this.Collection = function Collection(uri, recordObj, service) {
+ if (!service) {
+ throw new Error("Collection constructor requires a service.");
+ }
+
+ Resource.call(this, uri);
+
+ // This is a bit hacky, but gets the job done.
+ let res = service.resource(uri);
+ this.authenticator = res.authenticator;
+
+ this._recordObj = recordObj;
+ this._service = service;
+
+ this._full = false;
+ this._ids = null;
+ this._limit = 0;
+ this._older = 0;
+ this._newer = 0;
+ this._data = [];
+ // optional members used by batch upload operations.
+ this._batch = null;
+ this._commit = false;
+ // Used for batch download operations -- note that this is explicitly an
+ // opaque value and not (necessarily) a number.
+ this._offset = null;
+}
+Collection.prototype = {
+ __proto__: Resource.prototype,
+ _logName: "Sync.Collection",
+
+ _rebuildURL: function Coll__rebuildURL() {
+ // XXX should consider what happens if it's not a URL...
+ this.uri.QueryInterface(Ci.nsIURL);
+
+ let args = [];
+ if (this.older)
+ args.push('older=' + this.older);
+ else if (this.newer) {
+ args.push('newer=' + this.newer);
+ }
+ if (this.full)
+ args.push('full=1');
+ if (this.sort)
+ args.push('sort=' + this.sort);
+ if (this.ids != null)
+ args.push("ids=" + this.ids);
+ if (this.limit > 0 && this.limit != Infinity)
+ args.push("limit=" + this.limit);
+ if (this._batch)
+ args.push("batch=" + encodeURIComponent(this._batch));
+ if (this._commit)
+ args.push("commit=true");
+ if (this._offset)
+ args.push("offset=" + encodeURIComponent(this._offset));
+
+ this.uri.query = (args.length > 0)? '?' + args.join('&') : '';
+ },
+
+ // get full items
+ get full() { return this._full; },
+ set full(value) {
+ this._full = value;
+ this._rebuildURL();
+ },
+
+ // Apply the action to a certain set of ids
+ get ids() { return this._ids; },
+ set ids(value) {
+ this._ids = value;
+ this._rebuildURL();
+ },
+
+ // Limit how many records to get
+ get limit() { return this._limit; },
+ set limit(value) {
+ this._limit = value;
+ this._rebuildURL();
+ },
+
+ // get only items modified before some date
+ get older() { return this._older; },
+ set older(value) {
+ this._older = value;
+ this._rebuildURL();
+ },
+
+ // get only items modified since some date
+ get newer() { return this._newer; },
+ set newer(value) {
+ this._newer = value;
+ this._rebuildURL();
+ },
+
+ // get items sorted by some criteria. valid values:
+ // oldest (oldest first)
+ // newest (newest first)
+ // index
+ get sort() { return this._sort; },
+ set sort(value) {
+ this._sort = value;
+ this._rebuildURL();
+ },
+
+ get offset() { return this._offset; },
+ set offset(value) {
+ this._offset = value;
+ this._rebuildURL();
+ },
+
+ // Set information about the batch for this request.
+ get batch() { return this._batch; },
+ set batch(value) {
+ this._batch = value;
+ this._rebuildURL();
+ },
+
+ get commit() { return this._commit; },
+ set commit(value) {
+ this._commit = value && true;
+ this._rebuildURL();
+ },
+
+ // Similar to get(), but will page through the items `batchSize` at a time,
+ // deferring calling the record handler until we've gotten them all.
+ //
+ // Returns the last response processed, and doesn't run the record handler
+ // on any items if a non-success status is received while downloading the
+ // records (or if a network error occurs).
+ getBatched(batchSize = DEFAULT_DOWNLOAD_BATCH_SIZE) {
+ let totalLimit = Number(this.limit) || Infinity;
+ if (batchSize <= 0 || batchSize >= totalLimit) {
+ // Invalid batch sizes should arguably be an error, but they're easy to handle
+ return this.get();
+ }
+
+ if (!this.full) {
+ throw new Error("getBatched is unimplemented for guid-only GETs");
+ }
+
+ // _onComplete and _onProgress are reset after each `get` by AsyncResource.
+ // We overwrite _onRecord to something that stores the data in an array
+ // until the end.
+ let { _onComplete, _onProgress, _onRecord } = this;
+ let recordBuffer = [];
+ let resp;
+ try {
+ this._onRecord = r => recordBuffer.push(r);
+ let lastModifiedTime;
+ this.limit = batchSize;
+
+ do {
+ this._onProgress = _onProgress;
+ this._onComplete = _onComplete;
+ if (batchSize + recordBuffer.length > totalLimit) {
+ this.limit = totalLimit - recordBuffer.length;
+ }
+ this._log.trace("Performing batched GET", { limit: this.limit, offset: this.offset });
+ // Actually perform the request
+ resp = this.get();
+ if (!resp.success) {
+ break;
+ }
+
+ // Initialize last modified, or check that something broken isn't happening.
+ let lastModified = resp.headers["x-last-modified"];
+ if (!lastModifiedTime) {
+ lastModifiedTime = lastModified;
+ this.setHeader("X-If-Unmodified-Since", lastModified);
+ } else if (lastModified != lastModifiedTime) {
+ // Should be impossible -- We'd get a 412 in this case.
+ throw new Error("X-Last-Modified changed in the middle of a download batch! " +
+ `${lastModified} => ${lastModifiedTime}`)
+ }
+
+ // If this is missing, we're finished.
+ this.offset = resp.headers["x-weave-next-offset"];
+ } while (this.offset && totalLimit > recordBuffer.length);
+ } finally {
+ // Ensure we undo any temporary state so that subsequent calls to get()
+ // or getBatched() work properly. We do this before calling the record
+ // handler so that we can more convincingly pretend to be a normal get()
+ // call. Note: we're resetting these to the values they had before this
+ // function was called.
+ this._onRecord = _onRecord;
+ this._limit = totalLimit;
+ this._offset = null;
+ delete this._headers["x-if-unmodified-since"];
+ this._rebuildURL();
+ }
+ if (resp.success && Async.checkAppReady()) {
+ // call the original _onRecord (e.g. the user supplied record handler)
+ // for each record we've stored
+ for (let record of recordBuffer) {
+ this._onRecord(record);
+ }
+ }
+ return resp;
+ },
+
+ set recordHandler(onRecord) {
+ // Save this because onProgress is called with this as the ChannelListener
+ let coll = this;
+
+ // Switch to newline separated records for incremental parsing
+ coll.setHeader("Accept", "application/newlines");
+
+ this._onRecord = onRecord;
+
+ this._onProgress = function() {
+ let newline;
+ while ((newline = this._data.indexOf("\n")) > 0) {
+ // Split the json record from the rest of the data
+ let json = this._data.slice(0, newline);
+ this._data = this._data.slice(newline + 1);
+
+ // Deserialize a record from json and give it to the callback
+ let record = new coll._recordObj();
+ record.deserialize(json);
+ coll._onRecord(record);
+ }
+ };
+ },
+
+ // This object only supports posting via the postQueue object.
+ post() {
+ throw new Error("Don't directly post to a collection - use newPostQueue instead");
+ },
+
+ newPostQueue(log, timestamp, postCallback) {
+ let poster = (data, headers, batch, commit) => {
+ this.batch = batch;
+ this.commit = commit;
+ for (let [header, value] of headers) {
+ this.setHeader(header, value);
+ }
+ return Resource.prototype.post.call(this, data);
+ }
+ let getConfig = (name, defaultVal) => {
+ if (this._service.serverConfiguration && this._service.serverConfiguration.hasOwnProperty(name)) {
+ return this._service.serverConfiguration[name];
+ }
+ return defaultVal;
+ }
+
+ let config = {
+ max_post_bytes: getConfig("max_post_bytes", MAX_UPLOAD_BYTES),
+ max_post_records: getConfig("max_post_records", MAX_UPLOAD_RECORDS),
+
+ max_batch_bytes: getConfig("max_total_bytes", Infinity),
+ max_batch_records: getConfig("max_total_records", Infinity),
+ }
+
+ // Handle config edge cases
+ if (config.max_post_records <= 0) { config.max_post_records = MAX_UPLOAD_RECORDS; }
+ if (config.max_batch_records <= 0) { config.max_batch_records = Infinity; }
+ if (config.max_post_bytes <= 0) { config.max_post_bytes = MAX_UPLOAD_BYTES; }
+ if (config.max_batch_bytes <= 0) { config.max_batch_bytes = Infinity; }
+
+ // Max size of BSO payload is 256k. This assumes at most 4k of overhead,
+ // which sounds like plenty. If the server says it can't handle this, we
+ // might have valid records we can't sync, so we give up on syncing.
+ let requiredMax = 260 * 1024;
+ if (config.max_post_bytes < requiredMax) {
+ this._log.error("Server configuration max_post_bytes is too low", config);
+ throw new Error("Server configuration max_post_bytes is too low");
+ }
+
+ return new PostQueue(poster, timestamp, config, log, postCallback);
+ },
+};
+
+/* A helper to manage the posting of records while respecting the various
+ size limits.
+
+ This supports the concept of a server-side "batch". The general idea is:
+ * We queue as many records as allowed in memory, then make a single POST.
+ * This first POST (optionally) gives us a batch ID, which we use for
+ all subsequent posts, until...
+ * At some point we hit a batch-maximum, and jump through a few hoops to
+ commit the current batch (ie, all previous POSTs) and start a new one.
+ * Eventually commit the final batch.
+
+ In most cases we expect there to be exactly 1 batch consisting of possibly
+ multiple POSTs.
+*/
+function PostQueue(poster, timestamp, config, log, postCallback) {
+ // The "post" function we should use when it comes time to do the post.
+ this.poster = poster;
+ this.log = log;
+
+ // The config we use. We expect it to have fields "max_post_records",
+ // "max_batch_records", "max_post_bytes", and "max_batch_bytes"
+ this.config = config;
+
+ // The callback we make with the response when we do get around to making the
+ // post (which could be during any of the enqueue() calls or the final flush())
+ // This callback may be called multiple times and must not add new items to
+ // the queue.
+ // The second argument passed to this callback is a boolean value that is true
+ // if we're in the middle of a batch, and false if either the batch is
+ // complete, or it's a post to a server that does not understand batching.
+ this.postCallback = postCallback;
+
+ // The string where we are capturing the stringified version of the records
+ // queued so far. It will always be invalid JSON as it is always missing the
+ // closing bracket.
+ this.queued = "";
+
+ // The number of records we've queued so far but are yet to POST.
+ this.numQueued = 0;
+
+ // The number of records/bytes we've processed in previous POSTs for our
+ // current batch. Does *not* include records currently queued for the next POST.
+ this.numAlreadyBatched = 0;
+ this.bytesAlreadyBatched = 0;
+
+ // The ID of our current batch. Can be undefined (meaning we are yet to make
+ // the first post of a patch, so don't know if we have a batch), null (meaning
+ // we've made the first post but the server response indicated no batching
+ // semantics), otherwise we have made the first post and it holds the batch ID
+ // returned from the server.
+ this.batchID = undefined;
+
+ // Time used for X-If-Unmodified-Since -- should be the timestamp from the last GET.
+ this.lastModified = timestamp;
+}
+
+PostQueue.prototype = {
+ enqueue(record) {
+ // We want to ensure the record has a .toJSON() method defined - even
+ // though JSON.stringify() would implicitly call it, the stringify might
+ // still work even if it isn't defined, which isn't what we want.
+ let jsonRepr = record.toJSON();
+ if (!jsonRepr) {
+ throw new Error("You must only call this with objects that explicitly support JSON");
+ }
+ let bytes = JSON.stringify(jsonRepr);
+
+ // Do a flush if we can't add this record without exceeding our single-request
+ // limits, or without exceeding the total limit for a single batch.
+ let newLength = this.queued.length + bytes.length + 2; // extras for leading "[" / "," and trailing "]"
+
+ let maxAllowedBytes = Math.min(256 * 1024, this.config.max_post_bytes);
+
+ let postSizeExceeded = this.numQueued >= this.config.max_post_records ||
+ newLength >= maxAllowedBytes;
+
+ let batchSizeExceeded = (this.numQueued + this.numAlreadyBatched) >= this.config.max_batch_records ||
+ (newLength + this.bytesAlreadyBatched) >= this.config.max_batch_bytes;
+
+ let singleRecordTooBig = bytes.length + 2 > maxAllowedBytes;
+
+ if (postSizeExceeded || batchSizeExceeded) {
+ this.log.trace(`PostQueue flushing due to postSizeExceeded=${postSizeExceeded}, batchSizeExceeded=${batchSizeExceeded}` +
+ `, max_batch_bytes: ${this.config.max_batch_bytes}, max_post_bytes: ${this.config.max_post_bytes}`);
+
+ if (singleRecordTooBig) {
+ return { enqueued: false, error: new Error("Single record too large to submit to server") };
+ }
+
+ // We need to write the queue out before handling this one, but we only
+ // commit the batch (and thus start a new one) if the batch is full.
+ // Note that if a single record is too big for the batch or post, then
+ // the batch may be empty, and so we don't flush in that case.
+ if (this.numQueued) {
+ this.flush(batchSizeExceeded || singleRecordTooBig);
+ }
+ }
+ // Either a ',' or a '[' depending on whether this is the first record.
+ this.queued += this.numQueued ? "," : "[";
+ this.queued += bytes;
+ this.numQueued++;
+ return { enqueued: true };
+ },
+
+ flush(finalBatchPost) {
+ if (!this.queued) {
+ // nothing queued - we can't be in a batch, and something has gone very
+ // bad if we think we are.
+ if (this.batchID) {
+ throw new Error(`Flush called when no queued records but we are in a batch ${this.batchID}`);
+ }
+ return;
+ }
+ // the batch query-param and headers we'll send.
+ let batch;
+ let headers = [];
+ if (this.batchID === undefined) {
+ // First commit in a (possible) batch.
+ batch = "true";
+ } else if (this.batchID) {
+ // We have an existing batch.
+ batch = this.batchID;
+ } else {
+ // Not the first post and we know we have no batch semantics.
+ batch = null;
+ }
+
+ headers.push(["x-if-unmodified-since", this.lastModified]);
+
+ this.log.info(`Posting ${this.numQueued} records of ${this.queued.length+1} bytes with batch=${batch}`);
+ let queued = this.queued + "]";
+ if (finalBatchPost) {
+ this.bytesAlreadyBatched = 0;
+ this.numAlreadyBatched = 0;
+ } else {
+ this.bytesAlreadyBatched += queued.length;
+ this.numAlreadyBatched += this.numQueued;
+ }
+ this.queued = "";
+ this.numQueued = 0;
+ let response = this.poster(queued, headers, batch, !!(finalBatchPost && this.batchID !== null));
+
+ if (!response.success) {
+ this.log.trace("Server error response during a batch", response);
+ // not clear what we should do here - we expect the consumer of this to
+ // abort by throwing in the postCallback below.
+ return this.postCallback(response, !finalBatchPost);
+ }
+
+ if (finalBatchPost) {
+ this.log.trace("Committed batch", this.batchID);
+ this.batchID = undefined; // we are now in "first post for the batch" state.
+ this.lastModified = response.headers["x-last-modified"];
+ return this.postCallback(response, false);
+ }
+
+ if (response.status != 202) {
+ if (this.batchID) {
+ throw new Error("Server responded non-202 success code while a batch was in progress");
+ }
+ this.batchID = null; // no batch semantics are in place.
+ this.lastModified = response.headers["x-last-modified"];
+ return this.postCallback(response, false);
+ }
+
+ // this response is saying the server has batch semantics - we should
+ // always have a batch ID in the response.
+ let responseBatchID = response.obj.batch;
+ this.log.trace("Server responsed 202 with batch", responseBatchID);
+ if (!responseBatchID) {
+ this.log.error("Invalid server response: 202 without a batch ID", response);
+ throw new Error("Invalid server response: 202 without a batch ID");
+ }
+
+ if (this.batchID === undefined) {
+ this.batchID = responseBatchID;
+ if (!this.lastModified) {
+ this.lastModified = response.headers["x-last-modified"];
+ if (!this.lastModified) {
+ throw new Error("Batch response without x-last-modified");
+ }
+ }
+ }
+
+ if (this.batchID != responseBatchID) {
+ throw new Error(`Invalid client/server batch state - client has ${this.batchID}, server has ${responseBatchID}`);
+ }
+
+ this.postCallback(response, true);
+ },
+}