summaryrefslogtreecommitdiffstats
path: root/services/common/modules-testing/storageserver.js
diff options
context:
space:
mode:
authorMatt A. Tobin <mattatobin@localhost.localdomain>2018-02-02 04:16:08 -0500
committerMatt A. Tobin <mattatobin@localhost.localdomain>2018-02-02 04:16:08 -0500
commit5f8de423f190bbb79a62f804151bc24824fa32d8 (patch)
tree10027f336435511475e392454359edea8e25895d /services/common/modules-testing/storageserver.js
parent49ee0794b5d912db1f95dce6eb52d781dc210db5 (diff)
downloadUXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.gz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.lz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.tar.xz
UXP-5f8de423f190bbb79a62f804151bc24824fa32d8.zip
Add m-esr52 at 52.6.0
Diffstat (limited to 'services/common/modules-testing/storageserver.js')
-rw-r--r--services/common/modules-testing/storageserver.js1677
1 files changed, 1677 insertions, 0 deletions
diff --git a/services/common/modules-testing/storageserver.js b/services/common/modules-testing/storageserver.js
new file mode 100644
index 000000000..650ac307f
--- /dev/null
+++ b/services/common/modules-testing/storageserver.js
@@ -0,0 +1,1677 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * This file contains an implementation of the Storage Server in JavaScript.
+ *
+ * The server should not be used for any production purposes.
+ */
+
+var {classes: Cc, interfaces: Ci, utils: Cu} = Components;
+
+this.EXPORTED_SYMBOLS = [
+ "ServerBSO",
+ "StorageServerCallback",
+ "StorageServerCollection",
+ "StorageServer",
+ "storageServerForUsers",
+];
+
+Cu.import("resource://testing-common/httpd.js");
+Cu.import("resource://services-common/async.js");
+Cu.import("resource://gre/modules/Log.jsm");
+Cu.import("resource://services-common/utils.js");
+
+const STORAGE_HTTP_LOGGER = "Services.Common.Test.Server";
+const STORAGE_API_VERSION = "2.0";
+
+// Use the same method that record.js does, which mirrors the server.
+function new_timestamp() {
+ return Math.round(Date.now());
+}
+
+function isInteger(s) {
+ let re = /^[0-9]+$/;
+ return re.test(s);
+}
+
+function writeHttpBody(response, body) {
+ if (!body) {
+ return;
+ }
+
+ response.bodyOutputStream.write(body, body.length);
+}
+
+function sendMozSvcError(request, response, code) {
+ response.setStatusLine(request.httpVersion, 400, "Bad Request");
+ response.setHeader("Content-Type", "text/plain", false);
+ response.bodyOutputStream.write(code, code.length);
+}
+
+/**
+ * Represent a BSO on the server.
+ *
+ * A BSO is constructed from an ID, content, and a modified time.
+ *
+ * @param id
+ * (string) ID of the BSO being created.
+ * @param payload
+ * (strong|object) Payload for the BSO. Should ideally be a string. If
+ * an object is passed, it will be fed into JSON.stringify and that
+ * output will be set as the payload.
+ * @param modified
+ * (number) Milliseconds since UNIX epoch that the BSO was last
+ * modified. If not defined or null, the current time will be used.
+ */
+this.ServerBSO = function ServerBSO(id, payload, modified) {
+ if (!id) {
+ throw new Error("No ID for ServerBSO!");
+ }
+
+ if (!id.match(/^[a-zA-Z0-9_-]{1,64}$/)) {
+ throw new Error("BSO ID is invalid: " + id);
+ }
+
+ this._log = Log.repository.getLogger(STORAGE_HTTP_LOGGER);
+
+ this.id = id;
+ if (!payload) {
+ return;
+ }
+
+ CommonUtils.ensureMillisecondsTimestamp(modified);
+
+ if (typeof payload == "object") {
+ payload = JSON.stringify(payload);
+ }
+
+ this.payload = payload;
+ this.modified = modified || new_timestamp();
+}
+ServerBSO.prototype = {
+ FIELDS: [
+ "id",
+ "modified",
+ "payload",
+ "ttl",
+ "sortindex",
+ ],
+
+ toJSON: function toJSON() {
+ let obj = {};
+
+ for (let key of this.FIELDS) {
+ if (this[key] !== undefined) {
+ obj[key] = this[key];
+ }
+ }
+
+ return obj;
+ },
+
+ delete: function delete_() {
+ this.deleted = true;
+
+ delete this.payload;
+ delete this.modified;
+ },
+
+ /**
+ * Handler for GET requests for this BSO.
+ */
+ getHandler: function getHandler(request, response) {
+ let code = 200;
+ let status = "OK";
+ let body;
+
+ function sendResponse() {
+ response.setStatusLine(request.httpVersion, code, status);
+ writeHttpBody(response, body);
+ }
+
+ if (request.hasHeader("x-if-modified-since")) {
+ let headerModified = parseInt(request.getHeader("x-if-modified-since"),
+ 10);
+ CommonUtils.ensureMillisecondsTimestamp(headerModified);
+
+ if (headerModified >= this.modified) {
+ code = 304;
+ status = "Not Modified";
+
+ sendResponse();
+ return;
+ }
+ } else if (request.hasHeader("x-if-unmodified-since")) {
+ let requestModified = parseInt(request.getHeader("x-if-unmodified-since"),
+ 10);
+ let serverModified = this.modified;
+
+ if (serverModified > requestModified) {
+ code = 412;
+ status = "Precondition Failed";
+ sendResponse();
+ return;
+ }
+ }
+
+ if (!this.deleted) {
+ body = JSON.stringify(this.toJSON());
+ response.setHeader("Content-Type", "application/json", false);
+ response.setHeader("X-Last-Modified", "" + this.modified, false);
+ } else {
+ code = 404;
+ status = "Not Found";
+ }
+
+ sendResponse();
+ },
+
+ /**
+ * Handler for PUT requests for this BSO.
+ */
+ putHandler: function putHandler(request, response) {
+ if (request.hasHeader("Content-Type")) {
+ let ct = request.getHeader("Content-Type");
+ if (ct != "application/json") {
+ throw HTTP_415;
+ }
+ }
+
+ let input = CommonUtils.readBytesFromInputStream(request.bodyInputStream);
+ let parsed;
+ try {
+ parsed = JSON.parse(input);
+ } catch (ex) {
+ return sendMozSvcError(request, response, "8");
+ }
+
+ if (typeof(parsed) != "object") {
+ return sendMozSvcError(request, response, "8");
+ }
+
+ // Don't update if a conditional request fails preconditions.
+ if (request.hasHeader("x-if-unmodified-since")) {
+ let reqModified = parseInt(request.getHeader("x-if-unmodified-since"));
+
+ if (reqModified < this.modified) {
+ response.setStatusLine(request.httpVersion, 412, "Precondition Failed");
+ return;
+ }
+ }
+
+ let code, status;
+ if (this.payload) {
+ code = 204;
+ status = "No Content";
+ } else {
+ code = 201;
+ status = "Created";
+ }
+
+ // Alert when we see unrecognized fields.
+ for (let [key, value] of Object.entries(parsed)) {
+ switch (key) {
+ case "payload":
+ if (typeof(value) != "string") {
+ sendMozSvcError(request, response, "8");
+ return true;
+ }
+
+ this.payload = value;
+ break;
+
+ case "ttl":
+ if (!isInteger(value)) {
+ sendMozSvcError(request, response, "8");
+ return true;
+ }
+ this.ttl = parseInt(value, 10);
+ break;
+
+ case "sortindex":
+ if (!isInteger(value) || value.length > 9) {
+ sendMozSvcError(request, response, "8");
+ return true;
+ }
+ this.sortindex = parseInt(value, 10);
+ break;
+
+ case "id":
+ break;
+
+ default:
+ this._log.warn("Unexpected field in BSO record: " + key);
+ sendMozSvcError(request, response, "8");
+ return true;
+ }
+ }
+
+ this.modified = request.timestamp;
+ this.deleted = false;
+ response.setHeader("X-Last-Modified", "" + this.modified, false);
+
+ response.setStatusLine(request.httpVersion, code, status);
+ },
+};
+
+/**
+ * Represent a collection on the server.
+ *
+ * The '_bsos' attribute is a mapping of id -> ServerBSO objects.
+ *
+ * Note that if you want these records to be accessible individually,
+ * you need to register their handlers with the server separately, or use a
+ * containing HTTP server that will do so on your behalf.
+ *
+ * @param bsos
+ * An object mapping BSO IDs to ServerBSOs.
+ * @param acceptNew
+ * If true, POSTs to this collection URI will result in new BSOs being
+ * created and wired in on the fly.
+ * @param timestamp
+ * An optional timestamp value to initialize the modified time of the
+ * collection. This should be in the format returned by new_timestamp().
+ */
+this.StorageServerCollection =
+ function StorageServerCollection(bsos, acceptNew, timestamp=new_timestamp()) {
+ this._bsos = bsos || {};
+ this.acceptNew = acceptNew || false;
+
+ /*
+ * Track modified timestamp.
+ * We can't just use the timestamps of contained BSOs: an empty collection
+ * has a modified time.
+ */
+ CommonUtils.ensureMillisecondsTimestamp(timestamp);
+ this._timestamp = timestamp;
+
+ this._log = Log.repository.getLogger(STORAGE_HTTP_LOGGER);
+}
+StorageServerCollection.prototype = {
+ BATCH_MAX_COUNT: 100, // # of records.
+ BATCH_MAX_SIZE: 1024 * 1024, // # bytes.
+
+ _timestamp: null,
+
+ get timestamp() {
+ return this._timestamp;
+ },
+
+ set timestamp(timestamp) {
+ CommonUtils.ensureMillisecondsTimestamp(timestamp);
+ this._timestamp = timestamp;
+ },
+
+ get totalPayloadSize() {
+ let size = 0;
+ for (let bso of this.bsos()) {
+ size += bso.payload.length;
+ }
+
+ return size;
+ },
+
+ /**
+ * Convenience accessor for our BSO keys.
+ * Excludes deleted items, of course.
+ *
+ * @param filter
+ * A predicate function (applied to the ID and BSO) which dictates
+ * whether to include the BSO's ID in the output.
+ *
+ * @return an array of IDs.
+ */
+ keys: function keys(filter) {
+ let ids = [];
+ for (let [id, bso] of Object.entries(this._bsos)) {
+ if (!bso.deleted && (!filter || filter(id, bso))) {
+ ids.push(id);
+ }
+ }
+ return ids;
+ },
+
+ /**
+ * Convenience method to get an array of BSOs.
+ * Optionally provide a filter function.
+ *
+ * @param filter
+ * A predicate function, applied to the BSO, which dictates whether to
+ * include the BSO in the output.
+ *
+ * @return an array of ServerBSOs.
+ */
+ bsos: function bsos(filter) {
+ let os = [];
+ for (let [id, bso] of Object.entries(this._bsos)) {
+ if (!bso.deleted) {
+ os.push(bso);
+ }
+ }
+
+ if (!filter) {
+ return os;
+ }
+
+ return os.filter(filter);
+ },
+
+ /**
+ * Obtain a BSO by ID.
+ */
+ bso: function bso(id) {
+ return this._bsos[id];
+ },
+
+ /**
+ * Obtain the payload of a specific BSO.
+ *
+ * Raises if the specified BSO does not exist.
+ */
+ payload: function payload(id) {
+ return this.bso(id).payload;
+ },
+
+ /**
+ * Insert the provided BSO under its ID.
+ *
+ * @return the provided BSO.
+ */
+ insertBSO: function insertBSO(bso) {
+ return this._bsos[bso.id] = bso;
+ },
+
+ /**
+ * Insert the provided payload as part of a new ServerBSO with the provided
+ * ID.
+ *
+ * @param id
+ * The GUID for the BSO.
+ * @param payload
+ * The payload, as provided to the ServerBSO constructor.
+ * @param modified
+ * An optional modified time for the ServerBSO. If not specified, the
+ * current time will be used.
+ *
+ * @return the inserted BSO.
+ */
+ insert: function insert(id, payload, modified) {
+ return this.insertBSO(new ServerBSO(id, payload, modified));
+ },
+
+ /**
+ * Removes an object entirely from the collection.
+ *
+ * @param id
+ * (string) ID to remove.
+ */
+ remove: function remove(id) {
+ delete this._bsos[id];
+ },
+
+ _inResultSet: function _inResultSet(bso, options) {
+ if (!bso.payload) {
+ return false;
+ }
+
+ if (options.ids) {
+ if (options.ids.indexOf(bso.id) == -1) {
+ return false;
+ }
+ }
+
+ if (options.newer) {
+ if (bso.modified <= options.newer) {
+ return false;
+ }
+ }
+
+ if (options.older) {
+ if (bso.modified >= options.older) {
+ return false;
+ }
+ }
+
+ return true;
+ },
+
+ count: function count(options) {
+ options = options || {};
+ let c = 0;
+ for (let [id, bso] of Object.entries(this._bsos)) {
+ if (bso.modified && this._inResultSet(bso, options)) {
+ c++;
+ }
+ }
+ return c;
+ },
+
+ get: function get(options) {
+ let data = [];
+ for (let id in this._bsos) {
+ let bso = this._bsos[id];
+ if (!bso.modified) {
+ continue;
+ }
+
+ if (!this._inResultSet(bso, options)) {
+ continue;
+ }
+
+ data.push(bso);
+ }
+
+ if (options.sort) {
+ if (options.sort == "oldest") {
+ data.sort(function sortOldest(a, b) {
+ if (a.modified == b.modified) {
+ return 0;
+ }
+
+ return a.modified < b.modified ? -1 : 1;
+ });
+ } else if (options.sort == "newest") {
+ data.sort(function sortNewest(a, b) {
+ if (a.modified == b.modified) {
+ return 0;
+ }
+
+ return a.modified > b.modified ? -1 : 1;
+ });
+ } else if (options.sort == "index") {
+ data.sort(function sortIndex(a, b) {
+ if (a.sortindex == b.sortindex) {
+ return 0;
+ }
+
+ if (a.sortindex !== undefined && b.sortindex == undefined) {
+ return 1;
+ }
+
+ if (a.sortindex === undefined && b.sortindex !== undefined) {
+ return -1;
+ }
+
+ return a.sortindex > b.sortindex ? -1 : 1;
+ });
+ }
+ }
+
+ if (options.limit) {
+ data = data.slice(0, options.limit);
+ }
+
+ return data;
+ },
+
+ post: function post(input, timestamp) {
+ let success = [];
+ let failed = {};
+ let count = 0;
+ let size = 0;
+
+ // This will count records where we have an existing ServerBSO
+ // registered with us as successful and all other records as failed.
+ for (let record of input) {
+ count += 1;
+ if (count > this.BATCH_MAX_COUNT) {
+ failed[record.id] = "Max record count exceeded.";
+ continue;
+ }
+
+ if (typeof(record.payload) != "string") {
+ failed[record.id] = "Payload is not a string!";
+ continue;
+ }
+
+ size += record.payload.length;
+ if (size > this.BATCH_MAX_SIZE) {
+ failed[record.id] = "Payload max size exceeded!";
+ continue;
+ }
+
+ if (record.sortindex) {
+ if (!isInteger(record.sortindex)) {
+ failed[record.id] = "sortindex is not an integer.";
+ continue;
+ }
+
+ if (record.sortindex.length > 9) {
+ failed[record.id] = "sortindex is too long.";
+ continue;
+ }
+ }
+
+ if ("ttl" in record) {
+ if (!isInteger(record.ttl)) {
+ failed[record.id] = "ttl is not an integer.";
+ continue;
+ }
+ }
+
+ try {
+ let bso = this.bso(record.id);
+ if (!bso && this.acceptNew) {
+ this._log.debug("Creating BSO " + JSON.stringify(record.id) +
+ " on the fly.");
+ bso = new ServerBSO(record.id);
+ this.insertBSO(bso);
+ }
+ if (bso) {
+ bso.payload = record.payload;
+ bso.modified = timestamp;
+ bso.deleted = false;
+ success.push(record.id);
+
+ if (record.sortindex) {
+ bso.sortindex = parseInt(record.sortindex, 10);
+ }
+
+ } else {
+ failed[record.id] = "no bso configured";
+ }
+ } catch (ex) {
+ this._log.info("Exception when processing BSO", ex);
+ failed[record.id] = "Exception when processing.";
+ }
+ }
+ return {success: success, failed: failed};
+ },
+
+ delete: function delete_(options) {
+ options = options || {};
+
+ // Protocol 2.0 only allows the "ids" query string argument.
+ let keys = Object.keys(options).filter(function(k) {
+ return k != "ids";
+ });
+ if (keys.length) {
+ this._log.warn("Invalid query string parameter to collection delete: " +
+ keys.join(", "));
+ throw new Error("Malformed client request.");
+ }
+
+ if (options.ids && options.ids.length > this.BATCH_MAX_COUNT) {
+ throw HTTP_400;
+ }
+
+ let deleted = [];
+ for (let [id, bso] of Object.entries(this._bsos)) {
+ if (this._inResultSet(bso, options)) {
+ this._log.debug("Deleting " + JSON.stringify(bso));
+ deleted.push(bso.id);
+ bso.delete();
+ }
+ }
+ return deleted;
+ },
+
+ parseOptions: function parseOptions(request) {
+ let options = {};
+
+ for (let chunk of request.queryString.split("&")) {
+ if (!chunk) {
+ continue;
+ }
+ chunk = chunk.split("=");
+ let key = decodeURIComponent(chunk[0]);
+ if (chunk.length == 1) {
+ options[key] = "";
+ } else {
+ options[key] = decodeURIComponent(chunk[1]);
+ }
+ }
+
+ if (options.ids) {
+ options.ids = options.ids.split(",");
+ }
+
+ if (options.newer) {
+ if (!isInteger(options.newer)) {
+ throw HTTP_400;
+ }
+
+ CommonUtils.ensureMillisecondsTimestamp(options.newer);
+ options.newer = parseInt(options.newer, 10);
+ }
+
+ if (options.older) {
+ if (!isInteger(options.older)) {
+ throw HTTP_400;
+ }
+
+ CommonUtils.ensureMillisecondsTimestamp(options.older);
+ options.older = parseInt(options.older, 10);
+ }
+
+ if (options.limit) {
+ if (!isInteger(options.limit)) {
+ throw HTTP_400;
+ }
+
+ options.limit = parseInt(options.limit, 10);
+ }
+
+ return options;
+ },
+
+ getHandler: function getHandler(request, response) {
+ let options = this.parseOptions(request);
+ let data = this.get(options);
+
+ if (request.hasHeader("x-if-modified-since")) {
+ let requestModified = parseInt(request.getHeader("x-if-modified-since"),
+ 10);
+ let newestBSO = 0;
+ for (let bso of data) {
+ if (bso.modified > newestBSO) {
+ newestBSO = bso.modified;
+ }
+ }
+
+ if (requestModified >= newestBSO) {
+ response.setHeader("X-Last-Modified", "" + newestBSO);
+ response.setStatusLine(request.httpVersion, 304, "Not Modified");
+ return;
+ }
+ } else if (request.hasHeader("x-if-unmodified-since")) {
+ let requestModified = parseInt(request.getHeader("x-if-unmodified-since"),
+ 10);
+ let serverModified = this.timestamp;
+
+ if (serverModified > requestModified) {
+ response.setHeader("X-Last-Modified", "" + serverModified);
+ response.setStatusLine(request.httpVersion, 412, "Precondition Failed");
+ return;
+ }
+ }
+
+ if (options.full) {
+ data = data.map(function map(bso) {
+ return bso.toJSON();
+ });
+ } else {
+ data = data.map(function map(bso) {
+ return bso.id;
+ });
+ }
+
+ // application/json is default media type.
+ let newlines = false;
+ if (request.hasHeader("accept")) {
+ let accept = request.getHeader("accept");
+ if (accept == "application/newlines") {
+ newlines = true;
+ } else if (accept != "application/json") {
+ throw HTTP_406;
+ }
+ }
+
+ let body;
+ if (newlines) {
+ response.setHeader("Content-Type", "application/newlines", false);
+ let normalized = data.map(function map(d) {
+ return JSON.stringify(d);
+ });
+
+ body = normalized.join("\n") + "\n";
+ } else {
+ response.setHeader("Content-Type", "application/json", false);
+ body = JSON.stringify({items: data});
+ }
+
+ this._log.info("Records: " + data.length);
+ response.setHeader("X-Num-Records", "" + data.length, false);
+ response.setHeader("X-Last-Modified", "" + this.timestamp, false);
+ response.setStatusLine(request.httpVersion, 200, "OK");
+ response.bodyOutputStream.write(body, body.length);
+ },
+
+ postHandler: function postHandler(request, response) {
+ let options = this.parseOptions(request);
+
+ if (!request.hasHeader("content-type")) {
+ this._log.info("No Content-Type request header!");
+ throw HTTP_400;
+ }
+
+ let inputStream = request.bodyInputStream;
+ let inputBody = CommonUtils.readBytesFromInputStream(inputStream);
+ let input = [];
+
+ let inputMediaType = request.getHeader("content-type");
+ if (inputMediaType == "application/json") {
+ try {
+ input = JSON.parse(inputBody);
+ } catch (ex) {
+ this._log.info("JSON parse error on input body!");
+ throw HTTP_400;
+ }
+
+ if (!Array.isArray(input)) {
+ this._log.info("Input JSON type not an array!");
+ return sendMozSvcError(request, response, "8");
+ }
+ } else if (inputMediaType == "application/newlines") {
+ for (let line of inputBody.split("\n")) {
+ let record;
+ try {
+ record = JSON.parse(line);
+ } catch (ex) {
+ this._log.info("JSON parse error on line!");
+ return sendMozSvcError(request, response, "8");
+ }
+
+ input.push(record);
+ }
+ } else {
+ this._log.info("Unknown media type: " + inputMediaType);
+ throw HTTP_415;
+ }
+
+ if (this._ensureUnmodifiedSince(request, response)) {
+ return;
+ }
+
+ let res = this.post(input, request.timestamp);
+ let body = JSON.stringify(res);
+ response.setHeader("Content-Type", "application/json", false);
+ this.timestamp = request.timestamp;
+ response.setHeader("X-Last-Modified", "" + this.timestamp, false);
+
+ response.setStatusLine(request.httpVersion, "200", "OK");
+ response.bodyOutputStream.write(body, body.length);
+ },
+
+ deleteHandler: function deleteHandler(request, response) {
+ this._log.debug("Invoking StorageServerCollection.DELETE.");
+
+ let options = this.parseOptions(request);
+
+ if (this._ensureUnmodifiedSince(request, response)) {
+ return;
+ }
+
+ let deleted = this.delete(options);
+ response.deleted = deleted;
+ this.timestamp = request.timestamp;
+
+ response.setStatusLine(request.httpVersion, 204, "No Content");
+ },
+
+ handler: function handler() {
+ let self = this;
+
+ return function(request, response) {
+ switch(request.method) {
+ case "GET":
+ return self.getHandler(request, response);
+
+ case "POST":
+ return self.postHandler(request, response);
+
+ case "DELETE":
+ return self.deleteHandler(request, response);
+
+ }
+
+ request.setHeader("Allow", "GET,POST,DELETE");
+ response.setStatusLine(request.httpVersion, 405, "Method Not Allowed");
+ };
+ },
+
+ _ensureUnmodifiedSince: function _ensureUnmodifiedSince(request, response) {
+ if (!request.hasHeader("x-if-unmodified-since")) {
+ return false;
+ }
+
+ let requestModified = parseInt(request.getHeader("x-if-unmodified-since"),
+ 10);
+ let serverModified = this.timestamp;
+
+ this._log.debug("Request modified time: " + requestModified +
+ "; Server modified time: " + serverModified);
+ if (serverModified <= requestModified) {
+ return false;
+ }
+
+ this._log.info("Conditional request rejected because client time older " +
+ "than collection timestamp.");
+ response.setStatusLine(request.httpVersion, 412, "Precondition Failed");
+ return true;
+ },
+};
+
+
+//===========================================================================//
+// httpd.js-based Storage server. //
+//===========================================================================//
+
+/**
+ * In general, the preferred way of using StorageServer is to directly
+ * introspect it. Callbacks are available for operations which are hard to
+ * verify through introspection, such as deletions.
+ *
+ * One of the goals of this server is to provide enough hooks for test code to
+ * find out what it needs without monkeypatching. Use this object as your
+ * prototype, and override as appropriate.
+ */
+this.StorageServerCallback = {
+ onCollectionDeleted: function onCollectionDeleted(user, collection) {},
+ onItemDeleted: function onItemDeleted(user, collection, bsoID) {},
+
+ /**
+ * Called at the top of every request.
+ *
+ * Allows the test to inspect the request. Hooks should be careful not to
+ * modify or change state of the request or they may impact future processing.
+ */
+ onRequest: function onRequest(request) {},
+};
+
+/**
+ * Construct a new test Storage server. Takes a callback object (e.g.,
+ * StorageServerCallback) as input.
+ */
+this.StorageServer = function StorageServer(callback) {
+ this.callback = callback || {__proto__: StorageServerCallback};
+ this.server = new HttpServer();
+ this.started = false;
+ this.users = {};
+ this.requestCount = 0;
+ this._log = Log.repository.getLogger(STORAGE_HTTP_LOGGER);
+
+ // Install our own default handler. This allows us to mess around with the
+ // whole URL space.
+ let handler = this.server._handler;
+ handler._handleDefault = this.handleDefault.bind(this, handler);
+}
+StorageServer.prototype = {
+ DEFAULT_QUOTA: 1024 * 1024, // # bytes.
+
+ server: null, // HttpServer.
+ users: null, // Map of username => {collections, password}.
+
+ /**
+ * If true, the server will allow any arbitrary user to be used.
+ *
+ * No authentication will be performed. Whatever user is detected from the
+ * URL or auth headers will be created (if needed) and used.
+ */
+ allowAllUsers: false,
+
+ /**
+ * Start the StorageServer's underlying HTTP server.
+ *
+ * @param port
+ * The numeric port on which to start. A falsy value implies to
+ * select any available port.
+ * @param cb
+ * A callback function (of no arguments) which is invoked after
+ * startup.
+ */
+ start: function start(port, cb) {
+ if (this.started) {
+ this._log.warn("Warning: server already started on " + this.port);
+ return;
+ }
+ if (!port) {
+ port = -1;
+ }
+ this.port = port;
+
+ try {
+ this.server.start(this.port);
+ this.port = this.server.identity.primaryPort;
+ this.started = true;
+ if (cb) {
+ cb();
+ }
+ } catch (ex) {
+ _("==========================================");
+ _("Got exception starting Storage HTTP server on port " + this.port);
+ _("Error: " + Log.exceptionStr(ex));
+ _("Is there a process already listening on port " + this.port + "?");
+ _("==========================================");
+ do_throw(ex);
+ }
+ },
+
+ /**
+ * Start the server synchronously.
+ *
+ * @param port
+ * The numeric port on which to start. The default is to choose
+ * any available port.
+ */
+ startSynchronous: function startSynchronous(port=-1) {
+ let cb = Async.makeSpinningCallback();
+ this.start(port, cb);
+ cb.wait();
+ },
+
+ /**
+ * Stop the StorageServer's HTTP server.
+ *
+ * @param cb
+ * A callback function. Invoked after the server has been stopped.
+ *
+ */
+ stop: function stop(cb) {
+ if (!this.started) {
+ this._log.warn("StorageServer: Warning: server not running. Can't stop " +
+ "me now!");
+ return;
+ }
+
+ this.server.stop(cb);
+ this.started = false;
+ },
+
+ serverTime: function serverTime() {
+ return new_timestamp();
+ },
+
+ /**
+ * Create a new user, complete with an empty set of collections.
+ *
+ * @param username
+ * The username to use. An Error will be thrown if a user by that name
+ * already exists.
+ * @param password
+ * A password string.
+ *
+ * @return a user object, as would be returned by server.user(username).
+ */
+ registerUser: function registerUser(username, password) {
+ if (username in this.users) {
+ throw new Error("User already exists.");
+ }
+
+ if (!isFinite(parseInt(username))) {
+ throw new Error("Usernames must be numeric: " + username);
+ }
+
+ this._log.info("Registering new user with server: " + username);
+ this.users[username] = {
+ password: password,
+ collections: {},
+ quota: this.DEFAULT_QUOTA,
+ };
+ return this.user(username);
+ },
+
+ userExists: function userExists(username) {
+ return username in this.users;
+ },
+
+ getCollection: function getCollection(username, collection) {
+ return this.users[username].collections[collection];
+ },
+
+ _insertCollection: function _insertCollection(collections, collection, bsos) {
+ let coll = new StorageServerCollection(bsos, true);
+ coll.collectionHandler = coll.handler();
+ collections[collection] = coll;
+ return coll;
+ },
+
+ createCollection: function createCollection(username, collection, bsos) {
+ if (!(username in this.users)) {
+ throw new Error("Unknown user.");
+ }
+ let collections = this.users[username].collections;
+ if (collection in collections) {
+ throw new Error("Collection already exists.");
+ }
+ return this._insertCollection(collections, collection, bsos);
+ },
+
+ deleteCollection: function deleteCollection(username, collection) {
+ if (!(username in this.users)) {
+ throw new Error("Unknown user.");
+ }
+ delete this.users[username].collections[collection];
+ },
+
+ /**
+ * Accept a map like the following:
+ * {
+ * meta: {global: {version: 1, ...}},
+ * crypto: {"keys": {}, foo: {bar: 2}},
+ * bookmarks: {}
+ * }
+ * to cause collections and BSOs to be created.
+ * If a collection already exists, no error is raised.
+ * If a BSO already exists, it will be updated to the new contents.
+ */
+ createContents: function createContents(username, collections) {
+ if (!(username in this.users)) {
+ throw new Error("Unknown user.");
+ }
+ let userCollections = this.users[username].collections;
+ for (let [id, contents] of Object.entries(collections)) {
+ let coll = userCollections[id] ||
+ this._insertCollection(userCollections, id);
+ for (let [bsoID, payload] of Object.entries(contents)) {
+ coll.insert(bsoID, payload);
+ }
+ }
+ },
+
+ /**
+ * Insert a BSO in an existing collection.
+ */
+ insertBSO: function insertBSO(username, collection, bso) {
+ if (!(username in this.users)) {
+ throw new Error("Unknown user.");
+ }
+ let userCollections = this.users[username].collections;
+ if (!(collection in userCollections)) {
+ throw new Error("Unknown collection.");
+ }
+ userCollections[collection].insertBSO(bso);
+ return bso;
+ },
+
+ /**
+ * Delete all of the collections for the named user.
+ *
+ * @param username
+ * The name of the affected user.
+ */
+ deleteCollections: function deleteCollections(username) {
+ if (!(username in this.users)) {
+ throw new Error("Unknown user.");
+ }
+ let userCollections = this.users[username].collections;
+ for (let name in userCollections) {
+ let coll = userCollections[name];
+ this._log.trace("Bulk deleting " + name + " for " + username + "...");
+ coll.delete({});
+ }
+ this.users[username].collections = {};
+ },
+
+ getQuota: function getQuota(username) {
+ if (!(username in this.users)) {
+ throw new Error("Unknown user.");
+ }
+
+ return this.users[username].quota;
+ },
+
+ /**
+ * Obtain the newest timestamp of all collections for a user.
+ */
+ newestCollectionTimestamp: function newestCollectionTimestamp(username) {
+ let collections = this.users[username].collections;
+ let newest = 0;
+ for (let name in collections) {
+ let collection = collections[name];
+ if (collection.timestamp > newest) {
+ newest = collection.timestamp;
+ }
+ }
+
+ return newest;
+ },
+
+ /**
+ * Compute the object that is returned for an info/collections request.
+ */
+ infoCollections: function infoCollections(username) {
+ let responseObject = {};
+ let colls = this.users[username].collections;
+ for (let coll in colls) {
+ responseObject[coll] = colls[coll].timestamp;
+ }
+ this._log.trace("StorageServer: info/collections returning " +
+ JSON.stringify(responseObject));
+ return responseObject;
+ },
+
+ infoCounts: function infoCounts(username) {
+ let data = {};
+ let collections = this.users[username].collections;
+ for (let [k, v] of Object.entries(collections)) {
+ let count = v.count();
+ if (!count) {
+ continue;
+ }
+
+ data[k] = count;
+ }
+
+ return data;
+ },
+
+ infoUsage: function infoUsage(username) {
+ let data = {};
+ let collections = this.users[username].collections;
+ for (let [k, v] of Object.entries(collections)) {
+ data[k] = v.totalPayloadSize;
+ }
+
+ return data;
+ },
+
+ infoQuota: function infoQuota(username) {
+ let total = 0;
+ let usage = this.infoUsage(username);
+ for (let key in usage) {
+ let value = usage[key];
+ total += value;
+ }
+
+ return {
+ quota: this.getQuota(username),
+ usage: total
+ };
+ },
+
+ /**
+ * Simple accessor to allow collective binding and abbreviation of a bunch of
+ * methods. Yay!
+ * Use like this:
+ *
+ * let u = server.user("john");
+ * u.collection("bookmarks").bso("abcdefg").payload; // Etc.
+ *
+ * @return a proxy for the user data stored in this server.
+ */
+ user: function user(username) {
+ let collection = this.getCollection.bind(this, username);
+ let createCollection = this.createCollection.bind(this, username);
+ let createContents = this.createContents.bind(this, username);
+ let modified = function (collectionName) {
+ return collection(collectionName).timestamp;
+ }
+ let deleteCollections = this.deleteCollections.bind(this, username);
+ let quota = this.getQuota.bind(this, username);
+ return {
+ collection: collection,
+ createCollection: createCollection,
+ createContents: createContents,
+ deleteCollections: deleteCollections,
+ modified: modified,
+ quota: quota,
+ };
+ },
+
+ _pruneExpired: function _pruneExpired() {
+ let now = Date.now();
+
+ for (let username in this.users) {
+ let user = this.users[username];
+ for (let name in user.collections) {
+ let collection = user.collections[name];
+ for (let bso of collection.bsos()) {
+ // ttl === 0 is a special case, so we can't simply !ttl.
+ if (typeof(bso.ttl) != "number") {
+ continue;
+ }
+
+ let ttlDate = bso.modified + (bso.ttl * 1000);
+ if (ttlDate < now) {
+ this._log.info("Deleting BSO because TTL expired: " + bso.id);
+ bso.delete();
+ }
+ }
+ }
+ }
+ },
+
+ /*
+ * Regular expressions for splitting up Storage request paths.
+ * Storage URLs are of the form:
+ * /$apipath/$version/$userid/$further
+ * where $further is usually:
+ * storage/$collection/$bso
+ * or
+ * storage/$collection
+ * or
+ * info/$op
+ *
+ * We assume for the sake of simplicity that $apipath is empty.
+ *
+ * N.B., we don't follow any kind of username spec here, because as far as I
+ * can tell there isn't one. See Bug 689671. Instead we follow the Python
+ * server code.
+ *
+ * Path: [all, version, first, rest]
+ * Storage: [all, collection?, id?]
+ */
+ pathRE: /^\/([0-9]+(?:\.[0-9]+)?)(?:\/([0-9]+)\/([^\/]+)(?:\/(.+))?)?$/,
+ storageRE: /^([-_a-zA-Z0-9]+)(?:\/([-_a-zA-Z0-9]+)\/?)?$/,
+
+ defaultHeaders: {},
+
+ /**
+ * HTTP response utility.
+ */
+ respond: function respond(req, resp, code, status, body, headers, timestamp) {
+ this._log.info("Response: " + code + " " + status);
+ resp.setStatusLine(req.httpVersion, code, status);
+ if (!headers) {
+ headers = this.defaultHeaders;
+ }
+ for (let header in headers) {
+ let value = headers[header];
+ resp.setHeader(header, value, false);
+ }
+
+ if (timestamp) {
+ resp.setHeader("X-Timestamp", "" + timestamp, false);
+ }
+
+ if (body) {
+ resp.bodyOutputStream.write(body, body.length);
+ }
+ },
+
+ /**
+ * This is invoked by the HttpServer. `this` is bound to the StorageServer;
+ * `handler` is the HttpServer's handler.
+ *
+ * TODO: need to use the correct Storage API response codes and errors here.
+ */
+ handleDefault: function handleDefault(handler, req, resp) {
+ this.requestCount++;
+ let timestamp = new_timestamp();
+ try {
+ this._handleDefault(handler, req, resp, timestamp);
+ } catch (e) {
+ if (e instanceof HttpError) {
+ this.respond(req, resp, e.code, e.description, "", {}, timestamp);
+ } else {
+ this._log.warn("StorageServer: handleDefault caught an error", e);
+ throw e;
+ }
+ }
+ },
+
+ _handleDefault: function _handleDefault(handler, req, resp, timestamp) {
+ let path = req.path;
+ if (req.queryString.length) {
+ path += "?" + req.queryString;
+ }
+
+ this._log.debug("StorageServer: Handling request: " + req.method + " " +
+ path);
+
+ if (this.callback.onRequest) {
+ this.callback.onRequest(req);
+ }
+
+ // Prune expired records for all users at top of request. This is the
+ // easiest way to process TTLs since all requests go through here.
+ this._pruneExpired();
+
+ req.timestamp = timestamp;
+ resp.setHeader("X-Timestamp", "" + timestamp, false);
+
+ let parts = this.pathRE.exec(req.path);
+ if (!parts) {
+ this._log.debug("StorageServer: Unexpected request: bad URL " + req.path);
+ throw HTTP_404;
+ }
+
+ let [all, version, userPath, first, rest] = parts;
+ if (version != STORAGE_API_VERSION) {
+ this._log.debug("StorageServer: Unknown version.");
+ throw HTTP_404;
+ }
+
+ let username;
+
+ // By default, the server requires users to be authenticated. When a
+ // request arrives, the user must have been previously configured and
+ // the request must have authentication. In "allow all users" mode, we
+ // take the username from the URL, create the user on the fly, and don't
+ // perform any authentication.
+ if (!this.allowAllUsers) {
+ // Enforce authentication.
+ if (!req.hasHeader("authorization")) {
+ this.respond(req, resp, 401, "Authorization Required", "{}", {
+ "WWW-Authenticate": 'Basic realm="secret"'
+ });
+ return;
+ }
+
+ let ensureUserExists = function ensureUserExists(username) {
+ if (this.userExists(username)) {
+ return;
+ }
+
+ this._log.info("StorageServer: Unknown user: " + username);
+ throw HTTP_401;
+ }.bind(this);
+
+ let auth = req.getHeader("authorization");
+ this._log.debug("Authorization: " + auth);
+
+ if (auth.indexOf("Basic ") == 0) {
+ let decoded = CommonUtils.safeAtoB(auth.substr(6));
+ this._log.debug("Decoded Basic Auth: " + decoded);
+ let [user, password] = decoded.split(":", 2);
+
+ if (!password) {
+ this._log.debug("Malformed HTTP Basic Authorization header: " + auth);
+ throw HTTP_400;
+ }
+
+ this._log.debug("Got HTTP Basic auth for user: " + user);
+ ensureUserExists(user);
+ username = user;
+
+ if (this.users[user].password != password) {
+ this._log.debug("StorageServer: Provided password is not correct.");
+ throw HTTP_401;
+ }
+ // TODO support token auth.
+ } else {
+ this._log.debug("Unsupported HTTP authorization type: " + auth);
+ throw HTTP_500;
+ }
+ // All users mode.
+ } else {
+ // Auto create user with dummy password.
+ if (!this.userExists(userPath)) {
+ this.registerUser(userPath, "DUMMY-PASSWORD-*&%#");
+ }
+
+ username = userPath;
+ }
+
+ // Hand off to the appropriate handler for this path component.
+ if (first in this.toplevelHandlers) {
+ let handler = this.toplevelHandlers[first];
+ try {
+ return handler.call(this, handler, req, resp, version, username, rest);
+ } catch (ex) {
+ this._log.warn("Got exception during request", ex);
+ throw ex;
+ }
+ }
+ this._log.debug("StorageServer: Unknown top-level " + first);
+ throw HTTP_404;
+ },
+
+ /**
+ * Collection of the handler methods we use for top-level path components.
+ */
+ toplevelHandlers: {
+ "storage": function handleStorage(handler, req, resp, version, username,
+ rest) {
+ let respond = this.respond.bind(this, req, resp);
+ if (!rest || !rest.length) {
+ this._log.debug("StorageServer: top-level storage " +
+ req.method + " request.");
+
+ if (req.method != "DELETE") {
+ respond(405, "Method Not Allowed", null, {"Allow": "DELETE"});
+ return;
+ }
+
+ this.user(username).deleteCollections();
+
+ respond(204, "No Content");
+ return;
+ }
+
+ let match = this.storageRE.exec(rest);
+ if (!match) {
+ this._log.warn("StorageServer: Unknown storage operation " + rest);
+ throw HTTP_404;
+ }
+ let [all, collection, bsoID] = match;
+ let coll = this.getCollection(username, collection);
+ let collectionExisted = !!coll;
+
+ switch (req.method) {
+ case "GET":
+ // Tried to GET on a collection that doesn't exist.
+ if (!coll) {
+ respond(404, "Not Found");
+ return;
+ }
+
+ // No BSO URL parameter goes to collection handler.
+ if (!bsoID) {
+ return coll.collectionHandler(req, resp);
+ }
+
+ // Handle non-existent BSO.
+ let bso = coll.bso(bsoID);
+ if (!bso) {
+ respond(404, "Not Found");
+ return;
+ }
+
+ // Proxy to BSO handler.
+ return bso.getHandler(req, resp);
+
+ case "DELETE":
+ // Collection doesn't exist.
+ if (!coll) {
+ respond(404, "Not Found");
+ return;
+ }
+
+ // Deleting a specific BSO.
+ if (bsoID) {
+ let bso = coll.bso(bsoID);
+
+ // BSO does not exist on the server. Nothing to do.
+ if (!bso) {
+ respond(404, "Not Found");
+ return;
+ }
+
+ if (req.hasHeader("x-if-unmodified-since")) {
+ let modified = parseInt(req.getHeader("x-if-unmodified-since"));
+ CommonUtils.ensureMillisecondsTimestamp(modified);
+
+ if (bso.modified > modified) {
+ respond(412, "Precondition Failed");
+ return;
+ }
+ }
+
+ bso.delete();
+ coll.timestamp = req.timestamp;
+ this.callback.onItemDeleted(username, collection, bsoID);
+ respond(204, "No Content");
+ return;
+ }
+
+ // Proxy to collection handler.
+ coll.collectionHandler(req, resp);
+
+ // Spot if this is a DELETE for some IDs, and don't blow away the
+ // whole collection!
+ //
+ // We already handled deleting the BSOs by invoking the deleted
+ // collection's handler. However, in the case of
+ //
+ // DELETE storage/foobar
+ //
+ // we also need to remove foobar from the collections map. This
+ // clause tries to differentiate the above request from
+ //
+ // DELETE storage/foobar?ids=foo,baz
+ //
+ // and do the right thing.
+ // TODO: less hacky method.
+ if (-1 == req.queryString.indexOf("ids=")) {
+ // When you delete the entire collection, we drop it.
+ this._log.debug("Deleting entire collection.");
+ delete this.users[username].collections[collection];
+ this.callback.onCollectionDeleted(username, collection);
+ }
+
+ // Notify of item deletion.
+ let deleted = resp.deleted || [];
+ for (let i = 0; i < deleted.length; ++i) {
+ this.callback.onItemDeleted(username, collection, deleted[i]);
+ }
+ return;
+
+ case "POST":
+ case "PUT":
+ // Auto-create collection if it doesn't exist.
+ if (!coll) {
+ coll = this.createCollection(username, collection);
+ }
+
+ try {
+ if (bsoID) {
+ let bso = coll.bso(bsoID);
+ if (!bso) {
+ this._log.trace("StorageServer: creating BSO " + collection +
+ "/" + bsoID);
+ try {
+ bso = coll.insert(bsoID);
+ } catch (ex) {
+ return sendMozSvcError(req, resp, "8");
+ }
+ }
+
+ bso.putHandler(req, resp);
+
+ coll.timestamp = req.timestamp;
+ return resp;
+ }
+
+ return coll.collectionHandler(req, resp);
+ } catch (ex) {
+ if (ex instanceof HttpError) {
+ if (!collectionExisted) {
+ this.deleteCollection(username, collection);
+ }
+ }
+
+ throw ex;
+ }
+
+ default:
+ throw new Error("Request method " + req.method + " not implemented.");
+ }
+ },
+
+ "info": function handleInfo(handler, req, resp, version, username, rest) {
+ switch (rest) {
+ case "collections":
+ return this.handleInfoCollections(req, resp, username);
+
+ case "collection_counts":
+ return this.handleInfoCounts(req, resp, username);
+
+ case "collection_usage":
+ return this.handleInfoUsage(req, resp, username);
+
+ case "quota":
+ return this.handleInfoQuota(req, resp, username);
+
+ default:
+ this._log.warn("StorageServer: Unknown info operation " + rest);
+ throw HTTP_404;
+ }
+ }
+ },
+
+ handleInfoConditional: function handleInfoConditional(request, response,
+ user) {
+ if (!request.hasHeader("x-if-modified-since")) {
+ return false;
+ }
+
+ let requestModified = request.getHeader("x-if-modified-since");
+ requestModified = parseInt(requestModified, 10);
+
+ let serverModified = this.newestCollectionTimestamp(user);
+
+ this._log.info("Server mtime: " + serverModified + "; Client modified: " +
+ requestModified);
+ if (serverModified > requestModified) {
+ return false;
+ }
+
+ this.respond(request, response, 304, "Not Modified", null, {
+ "X-Last-Modified": "" + serverModified
+ });
+
+ return true;
+ },
+
+ handleInfoCollections: function handleInfoCollections(request, response,
+ user) {
+ if (this.handleInfoConditional(request, response, user)) {
+ return;
+ }
+
+ let info = this.infoCollections(user);
+ let body = JSON.stringify(info);
+ this.respond(request, response, 200, "OK", body, {
+ "Content-Type": "application/json",
+ "X-Last-Modified": "" + this.newestCollectionTimestamp(user),
+ });
+ },
+
+ handleInfoCounts: function handleInfoCounts(request, response, user) {
+ if (this.handleInfoConditional(request, response, user)) {
+ return;
+ }
+
+ let counts = this.infoCounts(user);
+ let body = JSON.stringify(counts);
+
+ this.respond(request, response, 200, "OK", body, {
+ "Content-Type": "application/json",
+ "X-Last-Modified": "" + this.newestCollectionTimestamp(user),
+ });
+ },
+
+ handleInfoUsage: function handleInfoUsage(request, response, user) {
+ if (this.handleInfoConditional(request, response, user)) {
+ return;
+ }
+
+ let body = JSON.stringify(this.infoUsage(user));
+ this.respond(request, response, 200, "OK", body, {
+ "Content-Type": "application/json",
+ "X-Last-Modified": "" + this.newestCollectionTimestamp(user),
+ });
+ },
+
+ handleInfoQuota: function handleInfoQuota(request, response, user) {
+ if (this.handleInfoConditional(request, response, user)) {
+ return;
+ }
+
+ let body = JSON.stringify(this.infoQuota(user));
+ this.respond(request, response, 200, "OK", body, {
+ "Content-Type": "application/json",
+ "X-Last-Modified": "" + this.newestCollectionTimestamp(user),
+ });
+ },
+};
+
+/**
+ * Helper to create a storage server for a set of users.
+ *
+ * Each user is specified by a map of username to password.
+ */
+this.storageServerForUsers =
+ function storageServerForUsers(users, contents, callback) {
+ let server = new StorageServer(callback);
+ for (let [user, pass] of Object.entries(users)) {
+ server.registerUser(user, pass);
+ server.createContents(user, contents);
+ }
+ server.start();
+ return server;
+}