diff options
Diffstat (limited to 'toolkit/components/telemetry/TelemetryStorage.jsm')
| -rw-r--r-- | toolkit/components/telemetry/TelemetryStorage.jsm | 1882 |
1 files changed, 1882 insertions, 0 deletions
diff --git a/toolkit/components/telemetry/TelemetryStorage.jsm b/toolkit/components/telemetry/TelemetryStorage.jsm new file mode 100644 index 000000000..91cfc993d --- /dev/null +++ b/toolkit/components/telemetry/TelemetryStorage.jsm @@ -0,0 +1,1882 @@ +/* -*- js-indent-level: 2; indent-tabs-mode: nil -*- */ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +this.EXPORTED_SYMBOLS = ["TelemetryStorage"]; + +const Cc = Components.classes; +const Ci = Components.interfaces; +const Cr = Components.results; +const Cu = Components.utils; + +Cu.import("resource://gre/modules/AppConstants.jsm", this); +Cu.import("resource://gre/modules/Log.jsm"); +Cu.import("resource://gre/modules/Services.jsm", this); +Cu.import("resource://gre/modules/XPCOMUtils.jsm", this); +Cu.import("resource://gre/modules/osfile.jsm", this); +Cu.import("resource://gre/modules/Task.jsm", this); +Cu.import("resource://gre/modules/TelemetryUtils.jsm", this); +Cu.import("resource://gre/modules/Promise.jsm", this); +Cu.import("resource://gre/modules/Preferences.jsm", this); + +const LOGGER_NAME = "Toolkit.Telemetry"; +const LOGGER_PREFIX = "TelemetryStorage::"; + +const Telemetry = Services.telemetry; +const Utils = TelemetryUtils; + +// Compute the path of the pings archive on the first use. +const DATAREPORTING_DIR = "datareporting"; +const PINGS_ARCHIVE_DIR = "archived"; +const ABORTED_SESSION_FILE_NAME = "aborted-session-ping"; +const DELETION_PING_FILE_NAME = "pending-deletion-ping"; +const SESSION_STATE_FILE_NAME = "session-state.json"; + +XPCOMUtils.defineLazyGetter(this, "gDataReportingDir", function() { + return OS.Path.join(OS.Constants.Path.profileDir, DATAREPORTING_DIR); +}); +XPCOMUtils.defineLazyGetter(this, "gPingsArchivePath", function() { + return OS.Path.join(gDataReportingDir, PINGS_ARCHIVE_DIR); +}); +XPCOMUtils.defineLazyGetter(this, "gAbortedSessionFilePath", function() { + return OS.Path.join(gDataReportingDir, ABORTED_SESSION_FILE_NAME); +}); +XPCOMUtils.defineLazyGetter(this, "gDeletionPingFilePath", function() { + return OS.Path.join(gDataReportingDir, DELETION_PING_FILE_NAME); +}); +XPCOMUtils.defineLazyModuleGetter(this, "CommonUtils", + "resource://services-common/utils.js"); +// Maxmimum time, in milliseconds, archive pings should be retained. +const MAX_ARCHIVED_PINGS_RETENTION_MS = 60 * 24 * 60 * 60 * 1000; // 60 days + +// Maximum space the archive can take on disk (in Bytes). +const ARCHIVE_QUOTA_BYTES = 120 * 1024 * 1024; // 120 MB +// Maximum space the outgoing pings can take on disk, for Desktop (in Bytes). +const PENDING_PINGS_QUOTA_BYTES_DESKTOP = 15 * 1024 * 1024; // 15 MB +// Maximum space the outgoing pings can take on disk, for Mobile (in Bytes). +const PENDING_PINGS_QUOTA_BYTES_MOBILE = 1024 * 1024; // 1 MB + +// The maximum size a pending/archived ping can take on disk. +const PING_FILE_MAXIMUM_SIZE_BYTES = 1024 * 1024; // 1 MB + +// This special value is submitted when the archive is outside of the quota. +const ARCHIVE_SIZE_PROBE_SPECIAL_VALUE = 300; + +// This special value is submitted when the pending pings is outside of the quota, as +// we don't know the size of the pings above the quota. +const PENDING_PINGS_SIZE_PROBE_SPECIAL_VALUE = 17; + +const UUID_REGEX = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; + +/** + * This is thrown by |TelemetryStorage.loadPingFile| when reading the ping + * from the disk fails. + */ +function PingReadError(message="Error reading the ping file", becauseNoSuchFile = false) { + Error.call(this, message); + let error = new Error(); + this.name = "PingReadError"; + this.message = message; + this.stack = error.stack; + this.becauseNoSuchFile = becauseNoSuchFile; +} +PingReadError.prototype = Object.create(Error.prototype); +PingReadError.prototype.constructor = PingReadError; + +/** + * This is thrown by |TelemetryStorage.loadPingFile| when parsing the ping JSON + * content fails. + */ +function PingParseError(message="Error parsing ping content") { + Error.call(this, message); + let error = new Error(); + this.name = "PingParseError"; + this.message = message; + this.stack = error.stack; +} +PingParseError.prototype = Object.create(Error.prototype); +PingParseError.prototype.constructor = PingParseError; + +/** + * This is a policy object used to override behavior for testing. + */ +var Policy = { + now: () => new Date(), + getArchiveQuota: () => ARCHIVE_QUOTA_BYTES, + getPendingPingsQuota: () => (AppConstants.platform in ["android", "gonk"]) + ? PENDING_PINGS_QUOTA_BYTES_MOBILE + : PENDING_PINGS_QUOTA_BYTES_DESKTOP, +}; + +/** + * Wait for all promises in iterable to resolve or reject. This function + * always resolves its promise with undefined, and never rejects. + */ +function waitForAll(it) { + let dummy = () => {}; + let promises = Array.from(it, p => p.catch(dummy)); + return Promise.all(promises); +} + +/** + * Permanently intern the given string. This is mainly used for the ping.type + * strings that can be excessively duplicated in the _archivedPings map. Do not + * pass large or temporary strings to this function. + */ +function internString(str) { + return Symbol.keyFor(Symbol.for(str)); +} + +this.TelemetryStorage = { + get pingDirectoryPath() { + return OS.Path.join(OS.Constants.Path.profileDir, "saved-telemetry-pings"); + }, + + /** + * The maximum size a ping can have, in bytes. + */ + get MAXIMUM_PING_SIZE() { + return PING_FILE_MAXIMUM_SIZE_BYTES; + }, + /** + * Shutdown & block on any outstanding async activity in this module. + * + * @return {Promise} Promise that is resolved when shutdown is complete. + */ + shutdown: function() { + return TelemetryStorageImpl.shutdown(); + }, + + /** + * Save an archived ping to disk. + * + * @param {object} ping The ping data to archive. + * @return {promise} Promise that is resolved when the ping is successfully archived. + */ + saveArchivedPing: function(ping) { + return TelemetryStorageImpl.saveArchivedPing(ping); + }, + + /** + * Load an archived ping from disk. + * + * @param {string} id The pings id. + * @return {promise<object>} Promise that is resolved with the ping data. + */ + loadArchivedPing: function(id) { + return TelemetryStorageImpl.loadArchivedPing(id); + }, + + /** + * Get a list of info on the archived pings. + * This will scan the archive directory and grab basic data about the existing + * pings out of their filename. + * + * @return {promise<sequence<object>>} + */ + loadArchivedPingList: function() { + return TelemetryStorageImpl.loadArchivedPingList(); + }, + + /** + * Clean the pings archive by removing old pings. + * This will scan the archive directory. + * + * @return {Promise} Resolved when the cleanup task completes. + */ + runCleanPingArchiveTask: function() { + return TelemetryStorageImpl.runCleanPingArchiveTask(); + }, + + /** + * Run the task to enforce the pending pings quota. + * + * @return {Promise} Resolved when the cleanup task completes. + */ + runEnforcePendingPingsQuotaTask: function() { + return TelemetryStorageImpl.runEnforcePendingPingsQuotaTask(); + }, + + /** + * Run the task to remove all the pending pings (except the deletion ping). + * + * @return {Promise} Resolved when the pings are removed. + */ + runRemovePendingPingsTask: function() { + return TelemetryStorageImpl.runRemovePendingPingsTask(); + }, + + /** + * Reset the storage state in tests. + */ + reset: function() { + return TelemetryStorageImpl.reset(); + }, + + /** + * Test method that allows waiting on the archive clean task to finish. + */ + testCleanupTaskPromise: function() { + return (TelemetryStorageImpl._cleanArchiveTask || Promise.resolve()); + }, + + /** + * Test method that allows waiting on the pending pings quota task to finish. + */ + testPendingQuotaTaskPromise: function() { + return (TelemetryStorageImpl._enforcePendingPingsQuotaTask || Promise.resolve()); + }, + + /** + * Save a pending - outgoing - ping to disk and track it. + * + * @param {Object} ping The ping data. + * @return {Promise} Resolved when the ping was saved. + */ + savePendingPing: function(ping) { + return TelemetryStorageImpl.savePendingPing(ping); + }, + + /** + * Saves session data to disk. + * @param {Object} sessionData The session data. + * @return {Promise} Resolved when the data was saved. + */ + saveSessionData: function(sessionData) { + return TelemetryStorageImpl.saveSessionData(sessionData); + }, + + /** + * Loads session data from a session data file. + * @return {Promise<object>} Resolved with the session data in object form. + */ + loadSessionData: function() { + return TelemetryStorageImpl.loadSessionData(); + }, + + /** + * Load a pending ping from disk by id. + * + * @param {String} id The pings id. + * @return {Promise} Resolved with the loaded ping data. + */ + loadPendingPing: function(id) { + return TelemetryStorageImpl.loadPendingPing(id); + }, + + /** + * Remove a pending ping from disk by id. + * + * @param {String} id The pings id. + * @return {Promise} Resolved when the ping was removed. + */ + removePendingPing: function(id) { + return TelemetryStorageImpl.removePendingPing(id); + }, + + /** + * Returns a list of the currently pending pings in the format: + * { + * id: <string>, // The pings UUID. + * lastModificationDate: <number>, // Timestamp of the pings last modification. + * } + * This populates the list by scanning the disk. + * + * @return {Promise<sequence>} Resolved with the ping list. + */ + loadPendingPingList: function() { + return TelemetryStorageImpl.loadPendingPingList(); + }, + + /** + * Returns a list of the currently pending pings in the format: + * { + * id: <string>, // The pings UUID. + * lastModificationDate: <number>, // Timestamp of the pings last modification. + * } + * This does not scan pending pings on disk. + * + * @return {sequence} The current pending ping list. + */ + getPendingPingList: function() { + return TelemetryStorageImpl.getPendingPingList(); + }, + + /** + * Save an aborted-session ping to disk. This goes to a special location so + * it is not picked up as a pending ping. + * + * @param {object} ping The ping data to save. + * @return {promise} Promise that is resolved when the ping is successfully saved. + */ + saveAbortedSessionPing: function(ping) { + return TelemetryStorageImpl.saveAbortedSessionPing(ping); + }, + + /** + * Load the aborted-session ping from disk if present. + * + * @return {promise<object>} Promise that is resolved with the ping data if found. + * Otherwise returns null. + */ + loadAbortedSessionPing: function() { + return TelemetryStorageImpl.loadAbortedSessionPing(); + }, + + /** + * Save the deletion ping. + * @param ping The deletion ping. + * @return {Promise} A promise resolved when the ping is saved. + */ + saveDeletionPing: function(ping) { + return TelemetryStorageImpl.saveDeletionPing(ping); + }, + + /** + * Remove the deletion ping. + * @return {Promise} Resolved when the ping is deleted from the disk. + */ + removeDeletionPing: function() { + return TelemetryStorageImpl.removeDeletionPing(); + }, + + /** + * Check if the ping id identifies a deletion ping. + */ + isDeletionPing: function(aPingId) { + return TelemetryStorageImpl.isDeletionPing(aPingId); + }, + + /** + * Remove the aborted-session ping if present. + * + * @return {promise} Promise that is resolved once the ping is removed. + */ + removeAbortedSessionPing: function() { + return TelemetryStorageImpl.removeAbortedSessionPing(); + }, + + /** + * Save a single ping to a file. + * + * @param {object} ping The content of the ping to save. + * @param {string} file The destination file. + * @param {bool} overwrite If |true|, the file will be overwritten if it exists, + * if |false| the file will not be overwritten and no error will be reported if + * the file exists. + * @returns {promise} + */ + savePingToFile: function(ping, file, overwrite) { + return TelemetryStorageImpl.savePingToFile(ping, file, overwrite); + }, + + /** + * Save a ping to its file. + * + * @param {object} ping The content of the ping to save. + * @param {bool} overwrite If |true|, the file will be overwritten + * if it exists. + * @returns {promise} + */ + savePing: function(ping, overwrite) { + return TelemetryStorageImpl.savePing(ping, overwrite); + }, + + /** + * Add a ping to the saved pings directory so that it gets saved + * and sent along with other pings. + * + * @param {Object} pingData The ping object. + * @return {Promise} A promise resolved when the ping is saved to the pings directory. + */ + addPendingPing: function(pingData) { + return TelemetryStorageImpl.addPendingPing(pingData); + }, + + /** + * Remove the file for a ping + * + * @param {object} ping The ping. + * @returns {promise} + */ + cleanupPingFile: function(ping) { + return TelemetryStorageImpl.cleanupPingFile(ping); + }, + + /** + * The number of pending pings on disk. + */ + get pendingPingCount() { + return TelemetryStorageImpl.pendingPingCount; + }, + + /** + * Loads a ping file. + * @param {String} aFilePath The path of the ping file. + * @return {Promise<Object>} A promise resolved with the ping content or rejected if the + * ping contains invalid data. + */ + loadPingFile: Task.async(function* (aFilePath) { + return TelemetryStorageImpl.loadPingFile(aFilePath); + }), + + /** + * Remove FHR database files. This is temporary and will be dropped in + * the future. + * @return {Promise} Resolved when the database files are deleted. + */ + removeFHRDatabase: function() { + return TelemetryStorageImpl.removeFHRDatabase(); + }, + + /** + * Only used in tests, builds an archived ping path from the ping metadata. + * @param {String} aPingId The ping id. + * @param {Object} aDate The ping creation date. + * @param {String} aType The ping type. + * @return {String} The full path to the archived ping. + */ + _testGetArchivedPingPath: function(aPingId, aDate, aType) { + return getArchivedPingPath(aPingId, aDate, aType); + }, + + /** + * Only used in tests, this helper extracts ping metadata from a given filename. + * + * @param fileName {String} The filename. + * @return {Object} Null if the filename didn't match the expected form. + * Otherwise an object with the extracted data in the form: + * { timestamp: <number>, + * id: <string>, + * type: <string> } + */ + _testGetArchivedPingDataFromFileName: function(aFileName) { + return TelemetryStorageImpl._getArchivedPingDataFromFileName(aFileName); + }, + + /** + * Only used in tests, this helper allows cleaning up the pending ping storage. + */ + testClearPendingPings: function() { + return TelemetryStorageImpl.runRemovePendingPingsTask(); + } +}; + +/** + * This object allows the serialisation of asynchronous tasks. This is particularly + * useful to serialise write access to the disk in order to prevent race conditions + * to corrupt the data being written. + * We are using this to synchronize saving to the file that TelemetrySession persists + * its state in. + */ +function SaveSerializer() { + this._queuedOperations = []; + this._queuedInProgress = false; + this._log = Log.repository.getLoggerWithMessagePrefix(LOGGER_NAME, LOGGER_PREFIX); +} + +SaveSerializer.prototype = { + /** + * Enqueues an operation to a list to serialise their execution in order to prevent race + * conditions. Useful to serialise access to disk. + * + * @param {Function} aFunction The task function to enqueue. It must return a promise. + * @return {Promise} A promise resolved when the enqueued task completes. + */ + enqueueTask: function (aFunction) { + let promise = new Promise((resolve, reject) => + this._queuedOperations.push([aFunction, resolve, reject])); + + if (this._queuedOperations.length == 1) { + this._popAndPerformQueuedOperation(); + } + return promise; + }, + + /** + * Make sure to flush all the pending operations. + * @return {Promise} A promise resolved when all the pending operations have completed. + */ + flushTasks: function () { + let dummyTask = () => new Promise(resolve => resolve()); + return this.enqueueTask(dummyTask); + }, + + /** + * Pop a task from the queue, executes it and continue to the next one. + * This function recursively pops all the tasks. + */ + _popAndPerformQueuedOperation: function () { + if (!this._queuedOperations.length || this._queuedInProgress) { + return; + } + + this._log.trace("_popAndPerformQueuedOperation - Performing queued operation."); + let [func, resolve, reject] = this._queuedOperations.shift(); + let promise; + + try { + this._queuedInProgress = true; + promise = func(); + } catch (ex) { + this._log.warn("_popAndPerformQueuedOperation - Queued operation threw during execution. ", + ex); + this._queuedInProgress = false; + reject(ex); + this._popAndPerformQueuedOperation(); + return; + } + + if (!promise || typeof(promise.then) != "function") { + let msg = "Queued operation did not return a promise: " + func; + this._log.warn("_popAndPerformQueuedOperation - " + msg); + + this._queuedInProgress = false; + reject(new Error(msg)); + this._popAndPerformQueuedOperation(); + return; + } + + promise.then(result => { + this._queuedInProgress = false; + resolve(result); + this._popAndPerformQueuedOperation(); + }, + error => { + this._log.warn("_popAndPerformQueuedOperation - Failure when performing queued operation.", + error); + this._queuedInProgress = false; + reject(error); + this._popAndPerformQueuedOperation(); + }); + }, +}; + +var TelemetryStorageImpl = { + _logger: null, + // Used to serialize aborted session ping writes to disk. + _abortedSessionSerializer: new SaveSerializer(), + // Used to serialize deletion ping writes to disk. + _deletionPingSerializer: new SaveSerializer(), + // Used to serialize session state writes to disk. + _stateSaveSerializer: new SaveSerializer(), + + // Tracks the archived pings in a Map of (id -> {timestampCreated, type}). + // We use this to cache info on archived pings to avoid scanning the disk more than once. + _archivedPings: new Map(), + // A set of promises for pings currently being archived + _activelyArchiving: new Set(), + // Track the archive loading task to prevent multiple tasks from being executed. + _scanArchiveTask: null, + // Track the archive cleanup task. + _cleanArchiveTask: null, + // Whether we already scanned the archived pings on disk. + _scannedArchiveDirectory: false, + + // Track the pending ping removal task. + _removePendingPingsTask: null, + + // This tracks all the pending async ping save activity. + _activePendingPingSaves: new Set(), + + // Tracks the pending pings in a Map of (id -> {timestampCreated, type}). + // We use this to cache info on pending pings to avoid scanning the disk more than once. + _pendingPings: new Map(), + + // Track the pending pings enforce quota task. + _enforcePendingPingsQuotaTask: null, + + // Track the shutdown process to bail out of the clean up task quickly. + _shutdown: false, + + get _log() { + if (!this._logger) { + this._logger = Log.repository.getLoggerWithMessagePrefix(LOGGER_NAME, LOGGER_PREFIX); + } + + return this._logger; + }, + + /** + * Shutdown & block on any outstanding async activity in this module. + * + * @return {Promise} Promise that is resolved when shutdown is complete. + */ + shutdown: Task.async(function*() { + this._shutdown = true; + + // If the following tasks are still running, block on them. They will bail out as soon + // as possible. + yield this._abortedSessionSerializer.flushTasks().catch(ex => { + this._log.error("shutdown - failed to flush aborted-session writes", ex); + }); + + yield this._deletionPingSerializer.flushTasks().catch(ex => { + this._log.error("shutdown - failed to flush deletion ping writes", ex); + }); + + if (this._cleanArchiveTask) { + yield this._cleanArchiveTask.catch(ex => { + this._log.error("shutdown - the archive cleaning task failed", ex); + }); + } + + if (this._enforcePendingPingsQuotaTask) { + yield this._enforcePendingPingsQuotaTask.catch(ex => { + this._log.error("shutdown - the pending pings quota task failed", ex); + }); + } + + if (this._removePendingPingsTask) { + yield this._removePendingPingsTask.catch(ex => { + this._log.error("shutdown - the pending pings removal task failed", ex); + }); + } + + // Wait on pending pings still being saved. While OS.File should have shutdown + // blockers in place, we a) have seen weird errors being reported that might + // indicate a bad shutdown path and b) might have completion handlers hanging + // off the save operations that don't expect to be late in shutdown. + yield this.promisePendingPingSaves(); + }), + + /** + * Save an archived ping to disk. + * + * @param {object} ping The ping data to archive. + * @return {promise} Promise that is resolved when the ping is successfully archived. + */ + saveArchivedPing: function(ping) { + let promise = this._saveArchivedPingTask(ping); + this._activelyArchiving.add(promise); + promise.then((r) => { this._activelyArchiving.delete(promise); }, + (e) => { this._activelyArchiving.delete(promise); }); + return promise; + }, + + _saveArchivedPingTask: Task.async(function*(ping) { + const creationDate = new Date(ping.creationDate); + if (this._archivedPings.has(ping.id)) { + const data = this._archivedPings.get(ping.id); + if (data.timestampCreated > creationDate.getTime()) { + this._log.error("saveArchivedPing - trying to overwrite newer ping with the same id"); + return Promise.reject(new Error("trying to overwrite newer ping with the same id")); + } + this._log.warn("saveArchivedPing - overwriting older ping with the same id"); + } + + // Get the archived ping path and append the lz4 suffix to it (so we have 'jsonlz4'). + const filePath = getArchivedPingPath(ping.id, creationDate, ping.type) + "lz4"; + yield OS.File.makeDir(OS.Path.dirname(filePath), { ignoreExisting: true, + from: OS.Constants.Path.profileDir }); + yield this.savePingToFile(ping, filePath, /* overwrite*/ true, /* compressed*/ true); + + this._archivedPings.set(ping.id, { + timestampCreated: creationDate.getTime(), + type: internString(ping.type), + }); + + Telemetry.getHistogramById("TELEMETRY_ARCHIVE_SESSION_PING_COUNT").add(); + return undefined; + }), + + /** + * Load an archived ping from disk. + * + * @param {string} id The pings id. + * @return {promise<object>} Promise that is resolved with the ping data. + */ + loadArchivedPing: Task.async(function*(id) { + this._log.trace("loadArchivedPing - id: " + id); + + const data = this._archivedPings.get(id); + if (!data) { + this._log.trace("loadArchivedPing - no ping with id: " + id); + return Promise.reject(new Error("TelemetryStorage.loadArchivedPing - no ping with id " + id)); + } + + const path = getArchivedPingPath(id, new Date(data.timestampCreated), data.type); + const pathCompressed = path + "lz4"; + + // Purge pings which are too big. + let checkSize = function*(path) { + const fileSize = (yield OS.File.stat(path)).size; + if (fileSize > PING_FILE_MAXIMUM_SIZE_BYTES) { + Telemetry.getHistogramById("TELEMETRY_DISCARDED_ARCHIVED_PINGS_SIZE_MB") + .add(Math.floor(fileSize / 1024 / 1024)); + Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_ARCHIVED").add(); + yield OS.File.remove(path, {ignoreAbsent: true}); + throw new Error("loadArchivedPing - exceeded the maximum ping size: " + fileSize); + } + }; + + try { + // Try to load a compressed version of the archived ping first. + this._log.trace("loadArchivedPing - loading ping from: " + pathCompressed); + yield* checkSize(pathCompressed); + return yield this.loadPingFile(pathCompressed, /* compressed*/ true); + } catch (ex) { + if (!ex.becauseNoSuchFile) { + throw ex; + } + // If that fails, look for the uncompressed version. + this._log.trace("loadArchivedPing - compressed ping not found, loading: " + path); + yield* checkSize(path); + return yield this.loadPingFile(path, /* compressed*/ false); + } + }), + + /** + * Saves session data to disk. + */ + saveSessionData: function(sessionData) { + return this._stateSaveSerializer.enqueueTask(() => this._saveSessionData(sessionData)); + }, + + _saveSessionData: Task.async(function* (sessionData) { + let dataDir = OS.Path.join(OS.Constants.Path.profileDir, DATAREPORTING_DIR); + yield OS.File.makeDir(dataDir); + + let filePath = OS.Path.join(gDataReportingDir, SESSION_STATE_FILE_NAME); + try { + yield CommonUtils.writeJSON(sessionData, filePath); + } catch (e) { + this._log.error("_saveSessionData - Failed to write session data to " + filePath, e); + Telemetry.getHistogramById("TELEMETRY_SESSIONDATA_FAILED_SAVE").add(1); + } + }), + + /** + * Loads session data from the session data file. + * @return {Promise<Object>} A promise resolved with an object on success, + * with null otherwise. + */ + loadSessionData: function() { + return this._stateSaveSerializer.enqueueTask(() => this._loadSessionData()); + }, + + _loadSessionData: Task.async(function* () { + const dataFile = OS.Path.join(OS.Constants.Path.profileDir, DATAREPORTING_DIR, + SESSION_STATE_FILE_NAME); + let content; + try { + content = yield OS.File.read(dataFile, { encoding: "utf-8" }); + } catch (ex) { + this._log.info("_loadSessionData - can not load session data file", ex); + Telemetry.getHistogramById("TELEMETRY_SESSIONDATA_FAILED_LOAD").add(1); + return null; + } + + let data; + try { + data = JSON.parse(content); + } catch (ex) { + this._log.error("_loadSessionData - failed to parse session data", ex); + Telemetry.getHistogramById("TELEMETRY_SESSIONDATA_FAILED_PARSE").add(1); + return null; + } + + return data; + }), + + /** + * Remove an archived ping from disk. + * + * @param {string} id The pings id. + * @param {number} timestampCreated The pings creation timestamp. + * @param {string} type The pings type. + * @return {promise<object>} Promise that is resolved when the pings is removed. + */ + _removeArchivedPing: Task.async(function*(id, timestampCreated, type) { + this._log.trace("_removeArchivedPing - id: " + id + ", timestampCreated: " + timestampCreated + ", type: " + type); + const path = getArchivedPingPath(id, new Date(timestampCreated), type); + const pathCompressed = path + "lz4"; + + this._log.trace("_removeArchivedPing - removing ping from: " + path); + yield OS.File.remove(path, {ignoreAbsent: true}); + yield OS.File.remove(pathCompressed, {ignoreAbsent: true}); + // Remove the ping from the cache. + this._archivedPings.delete(id); + }), + + /** + * Clean the pings archive by removing old pings. + * + * @return {Promise} Resolved when the cleanup task completes. + */ + runCleanPingArchiveTask: function() { + // If there's an archive cleaning task already running, return it. + if (this._cleanArchiveTask) { + return this._cleanArchiveTask; + } + + // Make sure to clear |_cleanArchiveTask| once done. + let clear = () => this._cleanArchiveTask = null; + // Since there's no archive cleaning task running, start it. + this._cleanArchiveTask = this._cleanArchive().then(clear, clear); + return this._cleanArchiveTask; + }, + + /** + * Removes pings which are too old from the pings archive. + * @return {Promise} Resolved when the ping age check is complete. + */ + _purgeOldPings: Task.async(function*() { + this._log.trace("_purgeOldPings"); + + const nowDate = Policy.now(); + const startTimeStamp = nowDate.getTime(); + let dirIterator = new OS.File.DirectoryIterator(gPingsArchivePath); + let subdirs = (yield dirIterator.nextBatch()).filter(e => e.isDir); + dirIterator.close(); + + // Keep track of the newest removed month to update the cache, if needed. + let newestRemovedMonthTimestamp = null; + let evictedDirsCount = 0; + let maxDirAgeInMonths = 0; + + // Walk through the monthly subdirs of the form <YYYY-MM>/ + for (let dir of subdirs) { + if (this._shutdown) { + this._log.trace("_purgeOldPings - Terminating the clean up task due to shutdown"); + return; + } + + if (!isValidArchiveDir(dir.name)) { + this._log.warn("_purgeOldPings - skipping invalidly named subdirectory " + dir.path); + continue; + } + + const archiveDate = getDateFromArchiveDir(dir.name); + if (!archiveDate) { + this._log.warn("_purgeOldPings - skipping invalid subdirectory date " + dir.path); + continue; + } + + // If this archive directory is older than 180 days, remove it. + if ((startTimeStamp - archiveDate.getTime()) > MAX_ARCHIVED_PINGS_RETENTION_MS) { + try { + yield OS.File.removeDir(dir.path); + evictedDirsCount++; + + // Update the newest removed month. + newestRemovedMonthTimestamp = Math.max(archiveDate, newestRemovedMonthTimestamp); + } catch (ex) { + this._log.error("_purgeOldPings - Unable to remove " + dir.path, ex); + } + } else { + // We're not removing this directory, so record the age for the oldest directory. + const dirAgeInMonths = Utils.getElapsedTimeInMonths(archiveDate, nowDate); + maxDirAgeInMonths = Math.max(dirAgeInMonths, maxDirAgeInMonths); + } + } + + // Trigger scanning of the archived pings. + yield this.loadArchivedPingList(); + + // Refresh the cache: we could still skip this, but it's cheap enough to keep it + // to avoid introducing task dependencies. + if (newestRemovedMonthTimestamp) { + // Scan the archive cache for pings older than the newest directory pruned above. + for (let [id, info] of this._archivedPings) { + const timestampCreated = new Date(info.timestampCreated); + if (timestampCreated.getTime() > newestRemovedMonthTimestamp) { + continue; + } + // Remove outdated pings from the cache. + this._archivedPings.delete(id); + } + } + + const endTimeStamp = Policy.now().getTime(); + + // Save the time it takes to evict old directories and the eviction count. + Telemetry.getHistogramById("TELEMETRY_ARCHIVE_EVICTED_OLD_DIRS") + .add(evictedDirsCount); + Telemetry.getHistogramById("TELEMETRY_ARCHIVE_EVICTING_DIRS_MS") + .add(Math.ceil(endTimeStamp - startTimeStamp)); + Telemetry.getHistogramById("TELEMETRY_ARCHIVE_OLDEST_DIRECTORY_AGE") + .add(maxDirAgeInMonths); + }), + + /** + * Enforce a disk quota for the pings archive. + * @return {Promise} Resolved when the quota check is complete. + */ + _enforceArchiveQuota: Task.async(function*() { + this._log.trace("_enforceArchiveQuota"); + let startTimeStamp = Policy.now().getTime(); + + // Build an ordered list, from newer to older, of archived pings. + let pingList = Array.from(this._archivedPings, p => ({ + id: p[0], + timestampCreated: p[1].timestampCreated, + type: p[1].type, + })); + + pingList.sort((a, b) => b.timestampCreated - a.timestampCreated); + + // If our archive is too big, we should reduce it to reach 90% of the quota. + const SAFE_QUOTA = Policy.getArchiveQuota() * 0.9; + // The index of the last ping to keep. Pings older than this one will be deleted if + // the archive exceeds the quota. + let lastPingIndexToKeep = null; + let archiveSizeInBytes = 0; + + // Find the disk size of the archive. + for (let i = 0; i < pingList.length; i++) { + if (this._shutdown) { + this._log.trace("_enforceArchiveQuota - Terminating the clean up task due to shutdown"); + return; + } + + let ping = pingList[i]; + + // Get the size for this ping. + const fileSize = + yield getArchivedPingSize(ping.id, new Date(ping.timestampCreated), ping.type); + if (!fileSize) { + this._log.warn("_enforceArchiveQuota - Unable to find the size of ping " + ping.id); + continue; + } + + // Enforce a maximum file size limit on archived pings. + if (fileSize > PING_FILE_MAXIMUM_SIZE_BYTES) { + this._log.error("_enforceArchiveQuota - removing file exceeding size limit, size: " + fileSize); + // We just remove the ping from the disk, we don't bother removing it from pingList + // since it won't contribute to the quota. + yield this._removeArchivedPing(ping.id, ping.timestampCreated, ping.type) + .catch(e => this._log.error("_enforceArchiveQuota - failed to remove archived ping" + ping.id)); + Telemetry.getHistogramById("TELEMETRY_DISCARDED_ARCHIVED_PINGS_SIZE_MB") + .add(Math.floor(fileSize / 1024 / 1024)); + Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_ARCHIVED").add(); + continue; + } + + archiveSizeInBytes += fileSize; + + if (archiveSizeInBytes < SAFE_QUOTA) { + // We save the index of the last ping which is ok to keep in order to speed up ping + // pruning. + lastPingIndexToKeep = i; + } else if (archiveSizeInBytes > Policy.getArchiveQuota()) { + // Ouch, our ping archive is too big. Bail out and start pruning! + break; + } + } + + // Save the time it takes to check if the archive is over-quota. + Telemetry.getHistogramById("TELEMETRY_ARCHIVE_CHECKING_OVER_QUOTA_MS") + .add(Math.round(Policy.now().getTime() - startTimeStamp)); + + let submitProbes = (sizeInMB, evictedPings, elapsedMs) => { + Telemetry.getHistogramById("TELEMETRY_ARCHIVE_SIZE_MB").add(sizeInMB); + Telemetry.getHistogramById("TELEMETRY_ARCHIVE_EVICTED_OVER_QUOTA").add(evictedPings); + Telemetry.getHistogramById("TELEMETRY_ARCHIVE_EVICTING_OVER_QUOTA_MS").add(elapsedMs); + }; + + // Check if we're using too much space. If not, submit the archive size and bail out. + if (archiveSizeInBytes < Policy.getArchiveQuota()) { + submitProbes(Math.round(archiveSizeInBytes / 1024 / 1024), 0, 0); + return; + } + + this._log.info("_enforceArchiveQuota - archive size: " + archiveSizeInBytes + "bytes" + + ", safety quota: " + SAFE_QUOTA + "bytes"); + + startTimeStamp = Policy.now().getTime(); + let pingsToPurge = pingList.slice(lastPingIndexToKeep + 1); + + // Remove all the pings older than the last one which we are safe to keep. + for (let ping of pingsToPurge) { + if (this._shutdown) { + this._log.trace("_enforceArchiveQuota - Terminating the clean up task due to shutdown"); + return; + } + + // This list is guaranteed to be in order, so remove the pings at its + // beginning (oldest). + yield this._removeArchivedPing(ping.id, ping.timestampCreated, ping.type); + } + + const endTimeStamp = Policy.now().getTime(); + submitProbes(ARCHIVE_SIZE_PROBE_SPECIAL_VALUE, pingsToPurge.length, + Math.ceil(endTimeStamp - startTimeStamp)); + }), + + _cleanArchive: Task.async(function*() { + this._log.trace("cleanArchiveTask"); + + if (!(yield OS.File.exists(gPingsArchivePath))) { + return; + } + + // Remove pings older than 180 days. + try { + yield this._purgeOldPings(); + } catch (ex) { + this._log.error("_cleanArchive - There was an error removing old directories", ex); + } + + // Make sure we respect the archive disk quota. + yield this._enforceArchiveQuota(); + }), + + /** + * Run the task to enforce the pending pings quota. + * + * @return {Promise} Resolved when the cleanup task completes. + */ + runEnforcePendingPingsQuotaTask: Task.async(function*() { + // If there's a cleaning task already running, return it. + if (this._enforcePendingPingsQuotaTask) { + return this._enforcePendingPingsQuotaTask; + } + + // Since there's no quota enforcing task running, start it. + try { + this._enforcePendingPingsQuotaTask = this._enforcePendingPingsQuota(); + yield this._enforcePendingPingsQuotaTask; + } finally { + this._enforcePendingPingsQuotaTask = null; + } + return undefined; + }), + + /** + * Enforce a disk quota for the pending pings. + * @return {Promise} Resolved when the quota check is complete. + */ + _enforcePendingPingsQuota: Task.async(function*() { + this._log.trace("_enforcePendingPingsQuota"); + let startTimeStamp = Policy.now().getTime(); + + // Build an ordered list, from newer to older, of pending pings. + let pingList = Array.from(this._pendingPings, p => ({ + id: p[0], + lastModificationDate: p[1].lastModificationDate, + })); + + pingList.sort((a, b) => b.lastModificationDate - a.lastModificationDate); + + // If our pending pings directory is too big, we should reduce it to reach 90% of the quota. + const SAFE_QUOTA = Policy.getPendingPingsQuota() * 0.9; + // The index of the last ping to keep. Pings older than this one will be deleted if + // the pending pings directory size exceeds the quota. + let lastPingIndexToKeep = null; + let pendingPingsSizeInBytes = 0; + + // Find the disk size of the pending pings directory. + for (let i = 0; i < pingList.length; i++) { + if (this._shutdown) { + this._log.trace("_enforcePendingPingsQuota - Terminating the clean up task due to shutdown"); + return; + } + + let ping = pingList[i]; + + // Get the size for this ping. + const fileSize = yield getPendingPingSize(ping.id); + if (!fileSize) { + this._log.warn("_enforcePendingPingsQuota - Unable to find the size of ping " + ping.id); + continue; + } + + pendingPingsSizeInBytes += fileSize; + if (pendingPingsSizeInBytes < SAFE_QUOTA) { + // We save the index of the last ping which is ok to keep in order to speed up ping + // pruning. + lastPingIndexToKeep = i; + } else if (pendingPingsSizeInBytes > Policy.getPendingPingsQuota()) { + // Ouch, our pending pings directory size is too big. Bail out and start pruning! + break; + } + } + + // Save the time it takes to check if the pending pings are over-quota. + Telemetry.getHistogramById("TELEMETRY_PENDING_CHECKING_OVER_QUOTA_MS") + .add(Math.round(Policy.now().getTime() - startTimeStamp)); + + let recordHistograms = (sizeInMB, evictedPings, elapsedMs) => { + Telemetry.getHistogramById("TELEMETRY_PENDING_PINGS_SIZE_MB").add(sizeInMB); + Telemetry.getHistogramById("TELEMETRY_PENDING_PINGS_EVICTED_OVER_QUOTA").add(evictedPings); + Telemetry.getHistogramById("TELEMETRY_PENDING_EVICTING_OVER_QUOTA_MS").add(elapsedMs); + }; + + // Check if we're using too much space. If not, bail out. + if (pendingPingsSizeInBytes < Policy.getPendingPingsQuota()) { + recordHistograms(Math.round(pendingPingsSizeInBytes / 1024 / 1024), 0, 0); + return; + } + + this._log.info("_enforcePendingPingsQuota - size: " + pendingPingsSizeInBytes + "bytes" + + ", safety quota: " + SAFE_QUOTA + "bytes"); + + startTimeStamp = Policy.now().getTime(); + let pingsToPurge = pingList.slice(lastPingIndexToKeep + 1); + + // Remove all the pings older than the last one which we are safe to keep. + for (let ping of pingsToPurge) { + if (this._shutdown) { + this._log.trace("_enforcePendingPingsQuota - Terminating the clean up task due to shutdown"); + return; + } + + // This list is guaranteed to be in order, so remove the pings at its + // beginning (oldest). + yield this.removePendingPing(ping.id); + } + + const endTimeStamp = Policy.now().getTime(); + // We don't know the size of the pending pings directory if we are above the quota, + // since we stop scanning once we reach the quota. We use a special value to show + // this condition. + recordHistograms(PENDING_PINGS_SIZE_PROBE_SPECIAL_VALUE, pingsToPurge.length, + Math.ceil(endTimeStamp - startTimeStamp)); + }), + + /** + * Reset the storage state in tests. + */ + reset: function() { + this._shutdown = false; + this._scannedArchiveDirectory = false; + this._archivedPings = new Map(); + this._scannedPendingDirectory = false; + this._pendingPings = new Map(); + }, + + /** + * Get a list of info on the archived pings. + * This will scan the archive directory and grab basic data about the existing + * pings out of their filename. + * + * @return {promise<sequence<object>>} + */ + loadArchivedPingList: Task.async(function*() { + // If there's an archive loading task already running, return it. + if (this._scanArchiveTask) { + return this._scanArchiveTask; + } + + yield waitForAll(this._activelyArchiving); + + if (this._scannedArchiveDirectory) { + this._log.trace("loadArchivedPingList - Archive already scanned, hitting cache."); + return this._archivedPings; + } + + // Since there's no archive loading task running, start it. + let result; + try { + this._scanArchiveTask = this._scanArchive(); + result = yield this._scanArchiveTask; + } finally { + this._scanArchiveTask = null; + } + return result; + }), + + _scanArchive: Task.async(function*() { + this._log.trace("_scanArchive"); + + let submitProbes = (pingCount, dirCount) => { + Telemetry.getHistogramById("TELEMETRY_ARCHIVE_SCAN_PING_COUNT") + .add(pingCount); + Telemetry.getHistogramById("TELEMETRY_ARCHIVE_DIRECTORIES_COUNT") + .add(dirCount); + }; + + if (!(yield OS.File.exists(gPingsArchivePath))) { + submitProbes(0, 0); + return new Map(); + } + + let dirIterator = new OS.File.DirectoryIterator(gPingsArchivePath); + let subdirs = + (yield dirIterator.nextBatch()).filter(e => e.isDir).filter(e => isValidArchiveDir(e.name)); + dirIterator.close(); + + // Walk through the monthly subdirs of the form <YYYY-MM>/ + for (let dir of subdirs) { + this._log.trace("_scanArchive - checking in subdir: " + dir.path); + let pingIterator = new OS.File.DirectoryIterator(dir.path); + let pings = (yield pingIterator.nextBatch()).filter(e => !e.isDir); + pingIterator.close(); + + // Now process any ping files of the form "<timestamp>.<uuid>.<type>.[json|jsonlz4]". + for (let p of pings) { + // data may be null if the filename doesn't match the above format. + let data = this._getArchivedPingDataFromFileName(p.name); + if (!data) { + continue; + } + + // In case of conflicts, overwrite only with newer pings. + if (this._archivedPings.has(data.id)) { + const overwrite = data.timestamp > this._archivedPings.get(data.id).timestampCreated; + this._log.warn("_scanArchive - have seen this id before: " + data.id + + ", overwrite: " + overwrite); + if (!overwrite) { + continue; + } + + yield this._removeArchivedPing(data.id, data.timestampCreated, data.type) + .catch((e) => this._log.warn("_scanArchive - failed to remove ping", e)); + } + + this._archivedPings.set(data.id, { + timestampCreated: data.timestamp, + type: internString(data.type), + }); + } + } + + // Mark the archive as scanned, so we no longer hit the disk. + this._scannedArchiveDirectory = true; + // Update the ping and directories count histograms. + submitProbes(this._archivedPings.size, subdirs.length); + return this._archivedPings; + }), + + /** + * Save a single ping to a file. + * + * @param {object} ping The content of the ping to save. + * @param {string} file The destination file. + * @param {bool} overwrite If |true|, the file will be overwritten if it exists, + * if |false| the file will not be overwritten and no error will be reported if + * the file exists. + * @param {bool} [compress=false] If |true|, the file will use lz4 compression. Otherwise no + * compression will be used. + * @returns {promise} + */ + savePingToFile: Task.async(function*(ping, filePath, overwrite, compress = false) { + try { + this._log.trace("savePingToFile - path: " + filePath); + let pingString = JSON.stringify(ping); + let options = { tmpPath: filePath + ".tmp", noOverwrite: !overwrite }; + if (compress) { + options.compression = "lz4"; + } + yield OS.File.writeAtomic(filePath, pingString, options); + } catch (e) { + if (!e.becauseExists) { + throw e; + } + } + }), + + /** + * Save a ping to its file. + * + * @param {object} ping The content of the ping to save. + * @param {bool} overwrite If |true|, the file will be overwritten + * if it exists. + * @returns {promise} + */ + savePing: Task.async(function*(ping, overwrite) { + yield getPingDirectory(); + let file = pingFilePath(ping); + yield this.savePingToFile(ping, file, overwrite); + return file; + }), + + /** + * Add a ping to the saved pings directory so that it gets saved + * and sent along with other pings. + * Note: that the original ping file will not be modified. + * + * @param {Object} ping The ping object. + * @return {Promise} A promise resolved when the ping is saved to the pings directory. + */ + addPendingPing: function(ping) { + return this.savePendingPing(ping); + }, + + /** + * Remove the file for a ping + * + * @param {object} ping The ping. + * @returns {promise} + */ + cleanupPingFile: function(ping) { + return OS.File.remove(pingFilePath(ping)); + }, + + savePendingPing: function(ping) { + let p = this.savePing(ping, true).then((path) => { + this._pendingPings.set(ping.id, { + path: path, + lastModificationDate: Policy.now().getTime(), + }); + this._log.trace("savePendingPing - saved ping with id " + ping.id); + }); + this._trackPendingPingSaveTask(p); + return p; + }, + + loadPendingPing: Task.async(function*(id) { + this._log.trace("loadPendingPing - id: " + id); + let info = this._pendingPings.get(id); + if (!info) { + this._log.trace("loadPendingPing - unknown id " + id); + throw new Error("TelemetryStorage.loadPendingPing - no ping with id " + id); + } + + // Try to get the dimension of the ping. If that fails, update the histograms. + let fileSize = 0; + try { + fileSize = (yield OS.File.stat(info.path)).size; + } catch (e) { + if (!(e instanceof OS.File.Error) || !e.becauseNoSuchFile) { + throw e; + } + // Fall through and let |loadPingFile| report the error. + } + + // Purge pings which are too big. + if (fileSize > PING_FILE_MAXIMUM_SIZE_BYTES) { + yield this.removePendingPing(id); + Telemetry.getHistogramById("TELEMETRY_DISCARDED_PENDING_PINGS_SIZE_MB") + .add(Math.floor(fileSize / 1024 / 1024)); + Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_PENDING").add(); + throw new Error("loadPendingPing - exceeded the maximum ping size: " + fileSize); + } + + // Try to load the ping file. Update the related histograms on failure. + let ping; + try { + ping = yield this.loadPingFile(info.path, false); + } catch (e) { + // If we failed to load the ping, check what happened and update the histogram. + if (e instanceof PingReadError) { + Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_READ").add(); + } else if (e instanceof PingParseError) { + Telemetry.getHistogramById("TELEMETRY_PENDING_LOAD_FAILURE_PARSE").add(); + } + // Remove the ping from the cache, so we don't try to load it again. + this._pendingPings.delete(id); + // Then propagate the rejection. + throw e; + } + + return ping; + }), + + removePendingPing: function(id) { + let info = this._pendingPings.get(id); + if (!info) { + this._log.trace("removePendingPing - unknown id " + id); + return Promise.resolve(); + } + + this._log.trace("removePendingPing - deleting ping with id: " + id + + ", path: " + info.path); + this._pendingPings.delete(id); + return OS.File.remove(info.path).catch((ex) => + this._log.error("removePendingPing - failed to remove ping", ex)); + }, + + /** + * Track any pending ping save tasks through the promise passed here. + * This is needed to block on any outstanding ping save activity. + * + * @param {Object<Promise>} The save promise to track. + */ + _trackPendingPingSaveTask: function (promise) { + let clear = () => this._activePendingPingSaves.delete(promise); + promise.then(clear, clear); + this._activePendingPingSaves.add(promise); + }, + + /** + * Return a promise that allows to wait on pending pings being saved. + * @return {Object<Promise>} A promise resolved when all the pending pings save promises + * are resolved. + */ + promisePendingPingSaves: function () { + // Make sure to wait for all the promises, even if they reject. We don't need to log + // the failures here, as they are already logged elsewhere. + return waitForAll(this._activePendingPingSaves); + }, + + /** + * Run the task to remove all the pending pings (except the deletion ping). + * + * @return {Promise} Resolved when the pings are removed. + */ + runRemovePendingPingsTask: Task.async(function*() { + // If we already have a pending pings removal task active, return that. + if (this._removePendingPingsTask) { + return this._removePendingPingsTask; + } + + // Start the task to remove all pending pings. Also make sure to clear the task once done. + try { + this._removePendingPingsTask = this.removePendingPings(); + yield this._removePendingPingsTask; + } finally { + this._removePendingPingsTask = null; + } + return undefined; + }), + + removePendingPings: Task.async(function*() { + this._log.trace("removePendingPings - removing all pending pings"); + + // Wait on pending pings still being saved, so so we don't miss removing them. + yield this.promisePendingPingSaves(); + + // Individually remove existing pings, so we don't interfere with operations expecting + // the pending pings directory to exist. + const directory = TelemetryStorage.pingDirectoryPath; + let iter = new OS.File.DirectoryIterator(directory); + + try { + if (!(yield iter.exists())) { + this._log.trace("removePendingPings - the pending pings directory doesn't exist"); + return; + } + + let files = (yield iter.nextBatch()).filter(e => !e.isDir); + for (let file of files) { + try { + yield OS.File.remove(file.path); + } catch (ex) { + this._log.error("removePendingPings - failed to remove file " + file.path, ex); + continue; + } + } + } finally { + yield iter.close(); + } + }), + + loadPendingPingList: function() { + // If we already have a pending scanning task active, return that. + if (this._scanPendingPingsTask) { + return this._scanPendingPingsTask; + } + + if (this._scannedPendingDirectory) { + this._log.trace("loadPendingPingList - Pending already scanned, hitting cache."); + return Promise.resolve(this._buildPingList()); + } + + // Since there's no pending pings scan task running, start it. + // Also make sure to clear the task once done. + this._scanPendingPingsTask = this._scanPendingPings().then(pings => { + this._scanPendingPingsTask = null; + return pings; + }, ex => { + this._scanPendingPingsTask = null; + throw ex; + }); + return this._scanPendingPingsTask; + }, + + getPendingPingList: function() { + return this._buildPingList(); + }, + + _scanPendingPings: Task.async(function*() { + this._log.trace("_scanPendingPings"); + + let directory = TelemetryStorage.pingDirectoryPath; + let iter = new OS.File.DirectoryIterator(directory); + let exists = yield iter.exists(); + + try { + if (!exists) { + return []; + } + + let files = (yield iter.nextBatch()).filter(e => !e.isDir); + + for (let file of files) { + if (this._shutdown) { + return []; + } + + let info; + try { + info = yield OS.File.stat(file.path); + } catch (ex) { + this._log.error("_scanPendingPings - failed to stat file " + file.path, ex); + continue; + } + + // Enforce a maximum file size limit on pending pings. + if (info.size > PING_FILE_MAXIMUM_SIZE_BYTES) { + this._log.error("_scanPendingPings - removing file exceeding size limit " + file.path); + try { + yield OS.File.remove(file.path); + } catch (ex) { + this._log.error("_scanPendingPings - failed to remove file " + file.path, ex); + } finally { + Telemetry.getHistogramById("TELEMETRY_DISCARDED_PENDING_PINGS_SIZE_MB") + .add(Math.floor(info.size / 1024 / 1024)); + Telemetry.getHistogramById("TELEMETRY_PING_SIZE_EXCEEDED_PENDING").add(); + continue; + } + } + + let id = OS.Path.basename(file.path); + if (!UUID_REGEX.test(id)) { + this._log.trace("_scanPendingPings - filename is not a UUID: " + id); + id = Utils.generateUUID(); + } + + this._pendingPings.set(id, { + path: file.path, + lastModificationDate: info.lastModificationDate.getTime(), + }); + } + } finally { + yield iter.close(); + } + + // Explicitly load the deletion ping from its known path, if it's there. + if (yield OS.File.exists(gDeletionPingFilePath)) { + this._log.trace("_scanPendingPings - Adding pending deletion ping."); + // We can't get the ping id or the last modification date without hitting the disk. + // Since deletion has a special handling, we don't really need those. + this._pendingPings.set(Utils.generateUUID(), { + path: gDeletionPingFilePath, + lastModificationDate: Date.now(), + }); + } + + this._scannedPendingDirectory = true; + return this._buildPingList(); + }), + + _buildPingList: function() { + const list = Array.from(this._pendingPings, p => ({ + id: p[0], + lastModificationDate: p[1].lastModificationDate, + })); + + list.sort((a, b) => b.lastModificationDate - a.lastModificationDate); + return list; + }, + + get pendingPingCount() { + return this._pendingPings.size; + }, + + /** + * Loads a ping file. + * @param {String} aFilePath The path of the ping file. + * @param {Boolean} [aCompressed=false] If |true|, expects the file to be compressed using lz4. + * @return {Promise<Object>} A promise resolved with the ping content or rejected if the + * ping contains invalid data. + * @throws {PingReadError} There was an error while reading the ping file from the disk. + * @throws {PingParseError} There was an error while parsing the JSON content of the ping file. + */ + loadPingFile: Task.async(function* (aFilePath, aCompressed = false) { + let options = {}; + if (aCompressed) { + options.compression = "lz4"; + } + + let array; + try { + array = yield OS.File.read(aFilePath, options); + } catch (e) { + this._log.trace("loadPingfile - unreadable ping " + aFilePath, e); + throw new PingReadError(e.message, e.becauseNoSuchFile); + } + + let decoder = new TextDecoder(); + let string = decoder.decode(array); + let ping; + try { + ping = JSON.parse(string); + } catch (e) { + this._log.trace("loadPingfile - unparseable ping " + aFilePath, e); + yield OS.File.remove(aFilePath).catch((ex) => { + this._log.error("loadPingFile - failed removing unparseable ping file", ex); + }); + throw new PingParseError(e.message); + } + + return ping; + }), + + /** + * Archived pings are saved with file names of the form: + * "<timestamp>.<uuid>.<type>.[json|jsonlz4]" + * This helper extracts that data from a given filename. + * + * @param fileName {String} The filename. + * @return {Object} Null if the filename didn't match the expected form. + * Otherwise an object with the extracted data in the form: + * { timestamp: <number>, + * id: <string>, + * type: <string> } + */ + _getArchivedPingDataFromFileName: function(fileName) { + // Extract the parts. + let parts = fileName.split("."); + if (parts.length != 4) { + this._log.trace("_getArchivedPingDataFromFileName - should have 4 parts"); + return null; + } + + let [timestamp, uuid, type, extension] = parts; + if (extension != "json" && extension != "jsonlz4") { + this._log.trace("_getArchivedPingDataFromFileName - should have 'json' or 'jsonlz4' extension"); + return null; + } + + // Check for a valid timestamp. + timestamp = parseInt(timestamp); + if (Number.isNaN(timestamp)) { + this._log.trace("_getArchivedPingDataFromFileName - should have a valid timestamp"); + return null; + } + + // Check for a valid UUID. + if (!UUID_REGEX.test(uuid)) { + this._log.trace("_getArchivedPingDataFromFileName - should have a valid id"); + return null; + } + + // Check for a valid type string. + const typeRegex = /^[a-z0-9][a-z0-9-]+[a-z0-9]$/i; + if (!typeRegex.test(type)) { + this._log.trace("_getArchivedPingDataFromFileName - should have a valid type"); + return null; + } + + return { + timestamp: timestamp, + id: uuid, + type: type, + }; + }, + + saveAbortedSessionPing: Task.async(function*(ping) { + this._log.trace("saveAbortedSessionPing - ping path: " + gAbortedSessionFilePath); + yield OS.File.makeDir(gDataReportingDir, { ignoreExisting: true }); + + return this._abortedSessionSerializer.enqueueTask(() => + this.savePingToFile(ping, gAbortedSessionFilePath, true)); + }), + + loadAbortedSessionPing: Task.async(function*() { + let ping = null; + try { + ping = yield this.loadPingFile(gAbortedSessionFilePath); + } catch (ex) { + if (ex.becauseNoSuchFile) { + this._log.trace("loadAbortedSessionPing - no such file"); + } else { + this._log.error("loadAbortedSessionPing - error loading ping", ex) + } + } + return ping; + }), + + removeAbortedSessionPing: function() { + return this._abortedSessionSerializer.enqueueTask(Task.async(function*() { + try { + yield OS.File.remove(gAbortedSessionFilePath, { ignoreAbsent: false }); + this._log.trace("removeAbortedSessionPing - success"); + } catch (ex) { + if (ex.becauseNoSuchFile) { + this._log.trace("removeAbortedSessionPing - no such file"); + } else { + this._log.error("removeAbortedSessionPing - error removing ping", ex) + } + } + }.bind(this))); + }, + + /** + * Save the deletion ping. + * @param ping The deletion ping. + * @return {Promise} Resolved when the ping is saved. + */ + saveDeletionPing: Task.async(function*(ping) { + this._log.trace("saveDeletionPing - ping path: " + gDeletionPingFilePath); + yield OS.File.makeDir(gDataReportingDir, { ignoreExisting: true }); + + let p = this._deletionPingSerializer.enqueueTask(() => + this.savePingToFile(ping, gDeletionPingFilePath, true)); + this._trackPendingPingSaveTask(p); + return p; + }), + + /** + * Remove the deletion ping. + * @return {Promise} Resolved when the ping is deleted from the disk. + */ + removeDeletionPing: Task.async(function*() { + return this._deletionPingSerializer.enqueueTask(Task.async(function*() { + try { + yield OS.File.remove(gDeletionPingFilePath, { ignoreAbsent: false }); + this._log.trace("removeDeletionPing - success"); + } catch (ex) { + if (ex.becauseNoSuchFile) { + this._log.trace("removeDeletionPing - no such file"); + } else { + this._log.error("removeDeletionPing - error removing ping", ex) + } + } + }.bind(this))); + }), + + isDeletionPing: function(aPingId) { + this._log.trace("isDeletionPing - id: " + aPingId); + let pingInfo = this._pendingPings.get(aPingId); + if (!pingInfo) { + return false; + } + + if (pingInfo.path != gDeletionPingFilePath) { + return false; + } + + return true; + }, + + /** + * Remove FHR database files. This is temporary and will be dropped in + * the future. + * @return {Promise} Resolved when the database files are deleted. + */ + removeFHRDatabase: Task.async(function*() { + this._log.trace("removeFHRDatabase"); + + // Let's try to remove the FHR DB with the default filename first. + const FHR_DB_DEFAULT_FILENAME = "healthreport.sqlite"; + + // Even if it's uncommon, there may be 2 additional files: - a "write ahead log" + // (-wal) file and a "shared memory file" (-shm). We need to remove them as well. + let FILES_TO_REMOVE = [ + OS.Path.join(OS.Constants.Path.profileDir, FHR_DB_DEFAULT_FILENAME), + OS.Path.join(OS.Constants.Path.profileDir, FHR_DB_DEFAULT_FILENAME + "-wal"), + OS.Path.join(OS.Constants.Path.profileDir, FHR_DB_DEFAULT_FILENAME + "-shm"), + ]; + + // FHR could have used either the default DB file name or a custom one + // through this preference. + const FHR_DB_CUSTOM_FILENAME = + Preferences.get("datareporting.healthreport.dbName", undefined); + if (FHR_DB_CUSTOM_FILENAME) { + FILES_TO_REMOVE.push( + OS.Path.join(OS.Constants.Path.profileDir, FHR_DB_CUSTOM_FILENAME), + OS.Path.join(OS.Constants.Path.profileDir, FHR_DB_CUSTOM_FILENAME + "-wal"), + OS.Path.join(OS.Constants.Path.profileDir, FHR_DB_CUSTOM_FILENAME + "-shm")); + } + + for (let f of FILES_TO_REMOVE) { + yield OS.File.remove(f, {ignoreAbsent: true}) + .catch(e => this._log.error("removeFHRDatabase - failed to remove " + f, e)); + } + }), +}; + +// Utility functions + +function pingFilePath(ping) { + // Support legacy ping formats, who don't have an "id" field, but a "slug" field. + let pingIdentifier = (ping.slug) ? ping.slug : ping.id; + return OS.Path.join(TelemetryStorage.pingDirectoryPath, pingIdentifier); +} + +function getPingDirectory() { + return Task.spawn(function*() { + let directory = TelemetryStorage.pingDirectoryPath; + + if (!(yield OS.File.exists(directory))) { + yield OS.File.makeDir(directory, { unixMode: OS.Constants.S_IRWXU }); + } + + return directory; + }); +} + +/** + * Build the path to the archived ping. + * @param {String} aPingId The ping id. + * @param {Object} aDate The ping creation date. + * @param {String} aType The ping type. + * @return {String} The full path to the archived ping. + */ +function getArchivedPingPath(aPingId, aDate, aType) { + // Helper to pad the month to 2 digits, if needed (e.g. "1" -> "01"). + let addLeftPadding = value => (value < 10) ? ("0" + value) : value; + // Get the ping creation date and generate the archive directory to hold it. Note + // that getMonth returns a 0-based month, so we need to add an offset. + let archivedPingDir = OS.Path.join(gPingsArchivePath, + aDate.getFullYear() + '-' + addLeftPadding(aDate.getMonth() + 1)); + // Generate the archived ping file path as YYYY-MM/<TIMESTAMP>.UUID.type.json + let fileName = [aDate.getTime(), aPingId, aType, "json"].join("."); + return OS.Path.join(archivedPingDir, fileName); +} + +/** + * Get the size of the ping file on the disk. + * @return {Integer} The file size, in bytes, of the ping file or 0 on errors. + */ +var getArchivedPingSize = Task.async(function*(aPingId, aDate, aType) { + const path = getArchivedPingPath(aPingId, aDate, aType); + let filePaths = [ path + "lz4", path ]; + + for (let path of filePaths) { + try { + return (yield OS.File.stat(path)).size; + } catch (e) {} + } + + // That's odd, this ping doesn't seem to exist. + return 0; +}); + +/** + * Get the size of the pending ping file on the disk. + * @return {Integer} The file size, in bytes, of the ping file or 0 on errors. + */ +var getPendingPingSize = Task.async(function*(aPingId) { + const path = OS.Path.join(TelemetryStorage.pingDirectoryPath, aPingId) + try { + return (yield OS.File.stat(path)).size; + } catch (e) {} + + // That's odd, this ping doesn't seem to exist. + return 0; +}); + +/** + * Check if a directory name is in the "YYYY-MM" format. + * @param {String} aDirName The name of the pings archive directory. + * @return {Boolean} True if the directory name is in the right format, false otherwise. + */ +function isValidArchiveDir(aDirName) { + const dirRegEx = /^[0-9]{4}-[0-9]{2}$/; + return dirRegEx.test(aDirName); +} + +/** + * Gets a date object from an archive directory name. + * @param {String} aDirName The name of the pings archive directory. Must be in the YYYY-MM + * format. + * @return {Object} A Date object or null if the dir name is not valid. + */ +function getDateFromArchiveDir(aDirName) { + let [year, month] = aDirName.split("-"); + year = parseInt(year); + month = parseInt(month); + // Make sure to have sane numbers. + if (!Number.isFinite(month) || !Number.isFinite(year) || month < 1 || month > 12) { + return null; + } + return new Date(year, month - 1, 1, 0, 0, 0); +} |