summaryrefslogtreecommitdiffstats
path: root/toolkit/components/crashes/CrashManager.jsm
diff options
context:
space:
mode:
Diffstat (limited to 'toolkit/components/crashes/CrashManager.jsm')
-rw-r--r--toolkit/components/crashes/CrashManager.jsm1351
1 files changed, 1351 insertions, 0 deletions
diff --git a/toolkit/components/crashes/CrashManager.jsm b/toolkit/components/crashes/CrashManager.jsm
new file mode 100644
index 000000000..3aac33254
--- /dev/null
+++ b/toolkit/components/crashes/CrashManager.jsm
@@ -0,0 +1,1351 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
+const myScope = this;
+
+Cu.import("resource://gre/modules/Log.jsm", this);
+Cu.import("resource://gre/modules/osfile.jsm", this);
+Cu.import("resource://gre/modules/Promise.jsm", this);
+Cu.import("resource://gre/modules/Services.jsm", this);
+Cu.import("resource://gre/modules/Task.jsm", this);
+Cu.import("resource://gre/modules/Timer.jsm", this);
+Cu.import("resource://gre/modules/XPCOMUtils.jsm", this);
+Cu.import("resource://gre/modules/TelemetryController.jsm");
+Cu.import("resource://gre/modules/KeyValueParser.jsm");
+
+this.EXPORTED_SYMBOLS = [
+ "CrashManager",
+];
+
+/**
+ * How long to wait after application startup before crash event files are
+ * automatically aggregated.
+ *
+ * We defer aggregation for performance reasons, as we don't want too many
+ * services competing for I/O immediately after startup.
+ */
+const AGGREGATE_STARTUP_DELAY_MS = 57000;
+
+const MILLISECONDS_IN_DAY = 24 * 60 * 60 * 1000;
+
+// Converts Date to days since UNIX epoch.
+// This was copied from /services/metrics.storage.jsm. The implementation
+// does not account for leap seconds.
+function dateToDays(date) {
+ return Math.floor(date.getTime() / MILLISECONDS_IN_DAY);
+}
+
+
+/**
+ * A gateway to crash-related data.
+ *
+ * This type is generic and can be instantiated any number of times.
+ * However, most applications will typically only have one instance
+ * instantiated and that instance will point to profile and user appdata
+ * directories.
+ *
+ * Instances are created by passing an object with properties.
+ * Recognized properties are:
+ *
+ * pendingDumpsDir (string) (required)
+ * Where dump files that haven't been uploaded are located.
+ *
+ * submittedDumpsDir (string) (required)
+ * Where records of uploaded dumps are located.
+ *
+ * eventsDirs (array)
+ * Directories (defined as strings) where events files are written. This
+ * instance will collects events from files in the directories specified.
+ *
+ * storeDir (string)
+ * Directory we will use for our data store. This instance will write
+ * data files into the directory specified.
+ *
+ * telemetryStoreSizeKey (string)
+ * Telemetry histogram to report store size under.
+ */
+this.CrashManager = function (options) {
+ for (let k of ["pendingDumpsDir", "submittedDumpsDir", "eventsDirs",
+ "storeDir"]) {
+ if (!(k in options)) {
+ throw new Error("Required key not present in options: " + k);
+ }
+ }
+
+ this._log = Log.repository.getLogger("Crashes.CrashManager");
+
+ for (let k in options) {
+ let v = options[k];
+
+ switch (k) {
+ case "pendingDumpsDir":
+ this._pendingDumpsDir = v;
+ break;
+
+ case "submittedDumpsDir":
+ this._submittedDumpsDir = v;
+ break;
+
+ case "eventsDirs":
+ this._eventsDirs = v;
+ break;
+
+ case "storeDir":
+ this._storeDir = v;
+ break;
+
+ case "telemetryStoreSizeKey":
+ this._telemetryStoreSizeKey = v;
+ break;
+
+ default:
+ throw new Error("Unknown property in options: " + k);
+ }
+ }
+
+ // Promise for in-progress aggregation operation. We store it on the
+ // object so it can be returned for in-progress operations.
+ this._aggregatePromise = null;
+
+ // The CrashStore currently attached to this object.
+ this._store = null;
+
+ // A Task to retrieve the store. This is needed to avoid races when
+ // _getStore() is called multiple times in a short interval.
+ this._getStoreTask = null;
+
+ // The timer controlling the expiration of the CrashStore instance.
+ this._storeTimer = null;
+
+ // This is a semaphore that prevents the store from being freed by our
+ // timer-based resource freeing mechanism.
+ this._storeProtectedCount = 0;
+};
+
+this.CrashManager.prototype = Object.freeze({
+ // A crash in the main process.
+ PROCESS_TYPE_MAIN: "main",
+
+ // A crash in a content process.
+ PROCESS_TYPE_CONTENT: "content",
+
+ // A crash in a plugin process.
+ PROCESS_TYPE_PLUGIN: "plugin",
+
+ // A crash in a Gecko media plugin process.
+ PROCESS_TYPE_GMPLUGIN: "gmplugin",
+
+ // A crash in the GPU process.
+ PROCESS_TYPE_GPU: "gpu",
+
+ // A real crash.
+ CRASH_TYPE_CRASH: "crash",
+
+ // A hang.
+ CRASH_TYPE_HANG: "hang",
+
+ // Submission result values.
+ SUBMISSION_RESULT_OK: "ok",
+ SUBMISSION_RESULT_FAILED: "failed",
+
+ DUMP_REGEX: /^([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.dmp$/i,
+ SUBMITTED_REGEX: /^bp-(?:hr-)?([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\.txt$/i,
+ ALL_REGEX: /^(.*)$/,
+
+ // How long the store object should persist in memory before being
+ // automatically garbage collected.
+ STORE_EXPIRATION_MS: 60 * 1000,
+
+ // Number of days after which a crash with no activity will get purged.
+ PURGE_OLDER_THAN_DAYS: 180,
+
+ // The following are return codes for individual event file processing.
+ // File processed OK.
+ EVENT_FILE_SUCCESS: "ok",
+ // The event appears to be malformed.
+ EVENT_FILE_ERROR_MALFORMED: "malformed",
+ // The type of event is unknown.
+ EVENT_FILE_ERROR_UNKNOWN_EVENT: "unknown-event",
+
+ /**
+ * Obtain a list of all dumps pending upload.
+ *
+ * The returned value is a promise that resolves to an array of objects
+ * on success. Each element in the array has the following properties:
+ *
+ * id (string)
+ * The ID of the crash (a UUID).
+ *
+ * path (string)
+ * The filename of the crash (<UUID.dmp>)
+ *
+ * date (Date)
+ * When this dump was created
+ *
+ * The returned arry is sorted by the modified time of the file backing
+ * the entry, oldest to newest.
+ *
+ * @return Promise<Array>
+ */
+ pendingDumps: function () {
+ return this._getDirectoryEntries(this._pendingDumpsDir, this.DUMP_REGEX);
+ },
+
+ /**
+ * Obtain a list of all dump files corresponding to submitted crashes.
+ *
+ * The returned value is a promise that resolves to an Array of
+ * objects. Each object has the following properties:
+ *
+ * path (string)
+ * The path of the file this entry comes from.
+ *
+ * id (string)
+ * The crash UUID.
+ *
+ * date (Date)
+ * The (estimated) date this crash was submitted.
+ *
+ * The returned array is sorted by the modified time of the file backing
+ * the entry, oldest to newest.
+ *
+ * @return Promise<Array>
+ */
+ submittedDumps: function () {
+ return this._getDirectoryEntries(this._submittedDumpsDir,
+ this.SUBMITTED_REGEX);
+ },
+
+ /**
+ * Aggregates "loose" events files into the unified "database."
+ *
+ * This function should be called periodically to collect metadata from
+ * all events files into the central data store maintained by this manager.
+ *
+ * Once events have been stored in the backing store the corresponding
+ * source files are deleted.
+ *
+ * Only one aggregation operation is allowed to occur at a time. If this
+ * is called when an existing aggregation is in progress, the promise for
+ * the original call will be returned.
+ *
+ * @return promise<int> The number of event files that were examined.
+ */
+ aggregateEventsFiles: function () {
+ if (this._aggregatePromise) {
+ return this._aggregatePromise;
+ }
+
+ return this._aggregatePromise = Task.spawn(function* () {
+ if (this._aggregatePromise) {
+ return this._aggregatePromise;
+ }
+
+ try {
+ let unprocessedFiles = yield this._getUnprocessedEventsFiles();
+
+ let deletePaths = [];
+ let needsSave = false;
+
+ this._storeProtectedCount++;
+ for (let entry of unprocessedFiles) {
+ try {
+ let result = yield this._processEventFile(entry);
+
+ switch (result) {
+ case this.EVENT_FILE_SUCCESS:
+ needsSave = true;
+ // Fall through.
+
+ case this.EVENT_FILE_ERROR_MALFORMED:
+ deletePaths.push(entry.path);
+ break;
+
+ case this.EVENT_FILE_ERROR_UNKNOWN_EVENT:
+ break;
+
+ default:
+ Cu.reportError("Unhandled crash event file return code. Please " +
+ "file a bug: " + result);
+ }
+ } catch (ex) {
+ if (ex instanceof OS.File.Error) {
+ this._log.warn("I/O error reading " + entry.path, ex);
+ } else {
+ // We should never encounter an exception. This likely represents
+ // a coding error because all errors should be detected and
+ // converted to return codes.
+ //
+ // If we get here, report the error and delete the source file
+ // so we don't see it again.
+ Cu.reportError("Exception when processing crash event file: " +
+ Log.exceptionStr(ex));
+ deletePaths.push(entry.path);
+ }
+ }
+ }
+
+ if (needsSave) {
+ let store = yield this._getStore();
+ yield store.save();
+ }
+
+ for (let path of deletePaths) {
+ try {
+ yield OS.File.remove(path);
+ } catch (ex) {
+ this._log.warn("Error removing event file (" + path + ")", ex);
+ }
+ }
+
+ return unprocessedFiles.length;
+
+ } finally {
+ this._aggregatePromise = false;
+ this._storeProtectedCount--;
+ }
+ }.bind(this));
+ },
+
+ /**
+ * Prune old crash data.
+ *
+ * @param date
+ * (Date) The cutoff point for pruning. Crashes without data newer
+ * than this will be pruned.
+ */
+ pruneOldCrashes: function (date) {
+ return Task.spawn(function* () {
+ let store = yield this._getStore();
+ store.pruneOldCrashes(date);
+ yield store.save();
+ }.bind(this));
+ },
+
+ /**
+ * Run tasks that should be periodically performed.
+ */
+ runMaintenanceTasks: function () {
+ return Task.spawn(function* () {
+ yield this.aggregateEventsFiles();
+
+ let offset = this.PURGE_OLDER_THAN_DAYS * MILLISECONDS_IN_DAY;
+ yield this.pruneOldCrashes(new Date(Date.now() - offset));
+ }.bind(this));
+ },
+
+ /**
+ * Schedule maintenance tasks for some point in the future.
+ *
+ * @param delay
+ * (integer) Delay in milliseconds when maintenance should occur.
+ */
+ scheduleMaintenance: function (delay) {
+ let deferred = Promise.defer();
+
+ setTimeout(() => {
+ this.runMaintenanceTasks().then(deferred.resolve, deferred.reject);
+ }, delay);
+
+ return deferred.promise;
+ },
+
+ /**
+ * Record the occurrence of a crash.
+ *
+ * This method skips event files altogether and writes directly and
+ * immediately to the manager's data store.
+ *
+ * @param processType (string) One of the PROCESS_TYPE constants.
+ * @param crashType (string) One of the CRASH_TYPE constants.
+ * @param id (string) Crash ID. Likely a UUID.
+ * @param date (Date) When the crash occurred.
+ * @param metadata (dictionary) Crash metadata, may be empty.
+ *
+ * @return promise<null> Resolved when the store has been saved.
+ */
+ addCrash: function (processType, crashType, id, date, metadata) {
+ return Task.spawn(function* () {
+ let store = yield this._getStore();
+ if (store.addCrash(processType, crashType, id, date, metadata)) {
+ yield store.save();
+ }
+ }.bind(this));
+ },
+
+ /**
+ * Record the remote ID for a crash.
+ *
+ * @param crashID (string) Crash ID. Likely a UUID.
+ * @param remoteID (Date) Server/Breakpad ID.
+ *
+ * @return boolean True if the remote ID was recorded.
+ */
+ setRemoteCrashID: Task.async(function* (crashID, remoteID) {
+ let store = yield this._getStore();
+ if (store.setRemoteCrashID(crashID, remoteID)) {
+ yield store.save();
+ }
+ }),
+
+ /**
+ * Generate a submission ID for use with addSubmission{Attempt,Result}.
+ */
+ generateSubmissionID() {
+ return "sub-" + Cc["@mozilla.org/uuid-generator;1"]
+ .getService(Ci.nsIUUIDGenerator)
+ .generateUUID().toString().slice(1, -1);
+ },
+
+ /**
+ * Record the occurrence of a submission attempt for a crash.
+ *
+ * @param crashID (string) Crash ID. Likely a UUID.
+ * @param submissionID (string) Submission ID. Likely a UUID.
+ * @param date (Date) When the attempt occurred.
+ *
+ * @return boolean True if the attempt was recorded and false if not.
+ */
+ addSubmissionAttempt: Task.async(function* (crashID, submissionID, date) {
+ let store = yield this._getStore();
+ if (store.addSubmissionAttempt(crashID, submissionID, date)) {
+ yield store.save();
+ }
+ }),
+
+ /**
+ * Record the occurrence of a submission result for a crash.
+ *
+ * @param crashID (string) Crash ID. Likely a UUID.
+ * @param submissionID (string) Submission ID. Likely a UUID.
+ * @param date (Date) When the submission result was obtained.
+ * @param result (string) One of the SUBMISSION_RESULT constants.
+ *
+ * @return boolean True if the result was recorded and false if not.
+ */
+ addSubmissionResult: Task.async(function* (crashID, submissionID, date, result) {
+ let store = yield this._getStore();
+ if (store.addSubmissionResult(crashID, submissionID, date, result)) {
+ yield store.save();
+ }
+ }),
+
+ /**
+ * Set the classification of a crash.
+ *
+ * @param crashID (string) Crash ID. Likely a UUID.
+ * @param classifications (array) Crash classifications.
+ *
+ * @return boolean True if the data was recorded and false if not.
+ */
+ setCrashClassifications: Task.async(function* (crashID, classifications) {
+ let store = yield this._getStore();
+ if (store.setCrashClassifications(crashID, classifications)) {
+ yield store.save();
+ }
+ }),
+
+ /**
+ * Obtain the paths of all unprocessed events files.
+ *
+ * The promise-resolved array is sorted by file mtime, oldest to newest.
+ */
+ _getUnprocessedEventsFiles: function () {
+ return Task.spawn(function* () {
+ let entries = [];
+
+ for (let dir of this._eventsDirs) {
+ for (let e of yield this._getDirectoryEntries(dir, this.ALL_REGEX)) {
+ entries.push(e);
+ }
+ }
+
+ entries.sort((a, b) => { return a.date - b.date; });
+
+ return entries;
+ }.bind(this));
+ },
+
+ // See docs/crash-events.rst for the file format specification.
+ _processEventFile: function (entry) {
+ return Task.spawn(function* () {
+ let data = yield OS.File.read(entry.path);
+ let store = yield this._getStore();
+
+ let decoder = new TextDecoder();
+ data = decoder.decode(data);
+
+ let type, time;
+ let start = 0;
+ for (let i = 0; i < 2; i++) {
+ let index = data.indexOf("\n", start);
+ if (index == -1) {
+ return this.EVENT_FILE_ERROR_MALFORMED;
+ }
+
+ let sub = data.substring(start, index);
+ switch (i) {
+ case 0:
+ type = sub;
+ break;
+ case 1:
+ time = sub;
+ try {
+ time = parseInt(time, 10);
+ } catch (ex) {
+ return this.EVENT_FILE_ERROR_MALFORMED;
+ }
+ }
+
+ start = index + 1;
+ }
+ let date = new Date(time * 1000);
+ let payload = data.substring(start);
+
+ return this._handleEventFilePayload(store, entry, type, date, payload);
+ }.bind(this));
+ },
+
+ _handleEventFilePayload: function (store, entry, type, date, payload) {
+ // The payload types and formats are documented in docs/crash-events.rst.
+ // Do not change the format of an existing type. Instead, invent a new
+ // type.
+ // DO NOT ADD NEW TYPES WITHOUT DOCUMENTING!
+ let lines = payload.split("\n");
+
+ switch (type) {
+ case "crash.main.1":
+ if (lines.length > 1) {
+ this._log.warn("Multiple lines unexpected in payload for " +
+ entry.path);
+ return this.EVENT_FILE_ERROR_MALFORMED;
+ }
+ // fall-through
+ case "crash.main.2":
+ let crashID = lines[0];
+ let metadata = parseKeyValuePairsFromLines(lines.slice(1));
+ store.addCrash(this.PROCESS_TYPE_MAIN, this.CRASH_TYPE_CRASH,
+ crashID, date, metadata);
+
+ // If we have a saved environment, use it. Otherwise report
+ // the current environment.
+ let crashEnvironment = null;
+ let sessionId = null;
+ let stackTraces = null;
+ let reportMeta = Cu.cloneInto(metadata, myScope);
+ if ('TelemetryEnvironment' in reportMeta) {
+ try {
+ crashEnvironment = JSON.parse(reportMeta.TelemetryEnvironment);
+ } catch (e) {
+ Cu.reportError(e);
+ }
+ delete reportMeta.TelemetryEnvironment;
+ }
+ if ('TelemetrySessionId' in reportMeta) {
+ sessionId = reportMeta.TelemetrySessionId;
+ delete reportMeta.TelemetrySessionId;
+ }
+ if ('StackTraces' in reportMeta) {
+ try {
+ stackTraces = JSON.parse(reportMeta.StackTraces);
+ } catch (e) {
+ Cu.reportError(e);
+ }
+ delete reportMeta.StackTraces;
+ }
+ TelemetryController.submitExternalPing("crash",
+ {
+ version: 1,
+ crashDate: date.toISOString().slice(0, 10), // YYYY-MM-DD
+ sessionId: sessionId,
+ crashId: entry.id,
+ stackTraces: stackTraces,
+ metadata: reportMeta,
+ hasCrashEnvironment: (crashEnvironment !== null),
+ },
+ {
+ retentionDays: 180,
+ addClientId: true,
+ addEnvironment: true,
+ overrideEnvironment: crashEnvironment,
+ });
+ break;
+
+ case "crash.submission.1":
+ if (lines.length == 3) {
+ let [crashID, result, remoteID] = lines;
+ store.addCrash(this.PROCESS_TYPE_MAIN, this.CRASH_TYPE_CRASH,
+ crashID, date);
+
+ let submissionID = this.generateSubmissionID();
+ let succeeded = result === "true";
+ store.addSubmissionAttempt(crashID, submissionID, date);
+ store.addSubmissionResult(crashID, submissionID, date,
+ succeeded ? this.SUBMISSION_RESULT_OK :
+ this.SUBMISSION_RESULT_FAILED);
+ if (succeeded) {
+ store.setRemoteCrashID(crashID, remoteID);
+ }
+ } else {
+ return this.EVENT_FILE_ERROR_MALFORMED;
+ }
+ break;
+
+ default:
+ return this.EVENT_FILE_ERROR_UNKNOWN_EVENT;
+ }
+
+ return this.EVENT_FILE_SUCCESS;
+ },
+
+ /**
+ * The resolved promise is an array of objects with the properties:
+ *
+ * path -- String filename
+ * id -- regexp.match()[1] (likely the crash ID)
+ * date -- Date mtime of the file
+ */
+ _getDirectoryEntries: function (path, re) {
+ return Task.spawn(function* () {
+ try {
+ yield OS.File.stat(path);
+ } catch (ex) {
+ if (!(ex instanceof OS.File.Error) || !ex.becauseNoSuchFile) {
+ throw ex;
+ }
+ return [];
+ }
+
+ let it = new OS.File.DirectoryIterator(path);
+ let entries = [];
+
+ try {
+ yield it.forEach((entry, index, it) => {
+ if (entry.isDir) {
+ return undefined;
+ }
+
+ let match = re.exec(entry.name);
+ if (!match) {
+ return undefined;
+ }
+
+ return OS.File.stat(entry.path).then((info) => {
+ entries.push({
+ path: entry.path,
+ id: match[1],
+ date: info.lastModificationDate,
+ });
+ });
+ });
+ } finally {
+ it.close();
+ }
+
+ entries.sort((a, b) => { return a.date - b.date; });
+
+ return entries;
+ }.bind(this));
+ },
+
+ _getStore: function () {
+ if (this._getStoreTask) {
+ return this._getStoreTask;
+ }
+
+ return this._getStoreTask = Task.spawn(function* () {
+ try {
+ if (!this._store) {
+ yield OS.File.makeDir(this._storeDir, {
+ ignoreExisting: true,
+ unixMode: OS.Constants.libc.S_IRWXU,
+ });
+
+ let store = new CrashStore(this._storeDir,
+ this._telemetryStoreSizeKey);
+ yield store.load();
+
+ this._store = store;
+ this._storeTimer = Cc["@mozilla.org/timer;1"]
+ .createInstance(Ci.nsITimer);
+ }
+
+ // The application can go long periods without interacting with the
+ // store. Since the store takes up resources, we automatically "free"
+ // the store after inactivity so resources can be returned to the
+ // system. We do this via a timer and a mechanism that tracks when the
+ // store is being accessed.
+ this._storeTimer.cancel();
+
+ // This callback frees resources from the store unless the store
+ // is protected from freeing by some other process.
+ let timerCB = function () {
+ if (this._storeProtectedCount) {
+ this._storeTimer.initWithCallback(timerCB, this.STORE_EXPIRATION_MS,
+ this._storeTimer.TYPE_ONE_SHOT);
+ return;
+ }
+
+ // We kill the reference that we hold. GC will kill it later. If
+ // someone else holds a reference, that will prevent GC until that
+ // reference is gone.
+ this._store = null;
+ this._storeTimer = null;
+ }.bind(this);
+
+ this._storeTimer.initWithCallback(timerCB, this.STORE_EXPIRATION_MS,
+ this._storeTimer.TYPE_ONE_SHOT);
+
+ return this._store;
+ } finally {
+ this._getStoreTask = null;
+ }
+ }.bind(this));
+ },
+
+ /**
+ * Obtain information about all known crashes.
+ *
+ * Returns an array of CrashRecord instances. Instances are read-only.
+ */
+ getCrashes: function () {
+ return Task.spawn(function* () {
+ let store = yield this._getStore();
+
+ return store.crashes;
+ }.bind(this));
+ },
+
+ getCrashCountsByDay: function () {
+ return Task.spawn(function* () {
+ let store = yield this._getStore();
+
+ return store._countsByDay;
+ }.bind(this));
+ },
+});
+
+var gCrashManager;
+
+/**
+ * Interface to storage of crash data.
+ *
+ * This type handles storage of crash metadata. It exists as a separate type
+ * from the crash manager for performance reasons: since all crash metadata
+ * needs to be loaded into memory for access, we wish to easily dispose of all
+ * associated memory when this data is no longer needed. Having an isolated
+ * object whose references can easily be lost faciliates that simple disposal.
+ *
+ * When metadata is updated, the caller must explicitly persist the changes
+ * to disk. This prevents excessive I/O during updates.
+ *
+ * The store has a mechanism for ensuring it doesn't grow too large. A ceiling
+ * is placed on the number of daily events that can occur for events that can
+ * occur with relatively high frequency, notably plugin crashes and hangs
+ * (plugins can enter cycles where they repeatedly crash). If we've reached
+ * the high water mark and new data arrives, it's silently dropped.
+ * However, the count of actual events is always preserved. This allows
+ * us to report on the severity of problems beyond the storage threshold.
+ *
+ * Main process crashes are excluded from limits because they are both
+ * important and should be rare.
+ *
+ * @param storeDir (string)
+ * Directory the store should be located in.
+ * @param telemetrySizeKey (string)
+ * The telemetry histogram that should be used to store the size
+ * of the data file.
+ */
+function CrashStore(storeDir, telemetrySizeKey) {
+ this._storeDir = storeDir;
+ this._telemetrySizeKey = telemetrySizeKey;
+
+ this._storePath = OS.Path.join(storeDir, "store.json.mozlz4");
+
+ // Holds the read data from disk.
+ this._data = null;
+
+ // Maps days since UNIX epoch to a Map of event types to counts.
+ // This data structure is populated when the JSON file is loaded
+ // and is also updated when new events are added.
+ this._countsByDay = new Map();
+}
+
+CrashStore.prototype = Object.freeze({
+ // Maximum number of events to store per day. This establishes a
+ // ceiling on the per-type/per-day records that will be stored.
+ HIGH_WATER_DAILY_THRESHOLD: 100,
+
+ /**
+ * Reset all data.
+ */
+ reset() {
+ this._data = {
+ v: 1,
+ crashes: new Map(),
+ corruptDate: null,
+ };
+ this._countsByDay = new Map();
+ },
+
+ /**
+ * Load data from disk.
+ *
+ * @return Promise
+ */
+ load: function () {
+ return Task.spawn(function* () {
+ // Loading replaces data.
+ this.reset();
+
+ try {
+ let decoder = new TextDecoder();
+ let data = yield OS.File.read(this._storePath, {compression: "lz4"});
+ data = JSON.parse(decoder.decode(data));
+
+ if (data.corruptDate) {
+ this._data.corruptDate = new Date(data.corruptDate);
+ }
+
+ // actualCounts is used to validate that the derived counts by
+ // days stored in the payload matches up to actual data.
+ let actualCounts = new Map();
+
+ // In the past, submissions were stored as separate crash records
+ // with an id of e.g. "someID-submission". If we find IDs ending
+ // with "-submission", we will need to convert the data to be stored
+ // as actual submissions.
+ //
+ // The old way of storing submissions was used from FF33 - FF34. We
+ // drop this old data on the floor.
+ for (let id in data.crashes) {
+ if (id.endsWith("-submission")) {
+ continue;
+ }
+
+ let crash = data.crashes[id];
+ let denormalized = this._denormalize(crash);
+
+ denormalized.submissions = new Map();
+ if (crash.submissions) {
+ for (let submissionID in crash.submissions) {
+ let submission = crash.submissions[submissionID];
+ denormalized.submissions.set(submissionID,
+ this._denormalize(submission));
+ }
+ }
+
+ this._data.crashes.set(id, denormalized);
+
+ let key = dateToDays(denormalized.crashDate) + "-" + denormalized.type;
+ actualCounts.set(key, (actualCounts.get(key) || 0) + 1);
+
+ // If we have an OOM size, count the crash as an OOM in addition to
+ // being a main process crash.
+ if (denormalized.metadata &&
+ denormalized.metadata.OOMAllocationSize) {
+ let oomKey = key + "-oom";
+ actualCounts.set(oomKey, (actualCounts.get(oomKey) || 0) + 1);
+ }
+
+ }
+
+ // The validation in this loop is arguably not necessary. We perform
+ // it as a defense against unknown bugs.
+ for (let dayKey in data.countsByDay) {
+ let day = parseInt(dayKey, 10);
+ for (let type in data.countsByDay[day]) {
+ this._ensureCountsForDay(day);
+
+ let count = data.countsByDay[day][type];
+ let key = day + "-" + type;
+
+ // If the payload says we have data for a given day but we
+ // don't, the payload is wrong. Ignore it.
+ if (!actualCounts.has(key)) {
+ continue;
+ }
+
+ // If we encountered more data in the payload than what the
+ // data structure says, use the proper value.
+ count = Math.max(count, actualCounts.get(key));
+
+ this._countsByDay.get(day).set(type, count);
+ }
+ }
+ } catch (ex) {
+ // Missing files (first use) are allowed.
+ if (!(ex instanceof OS.File.Error) || !ex.becauseNoSuchFile) {
+ // If we can't load for any reason, mark a corrupt date in the instance
+ // and swallow the error.
+ //
+ // The marking of a corrupted file is intentionally not persisted to
+ // disk yet. Instead, we wait until the next save(). This is to give
+ // non-permanent failures the opportunity to recover on their own.
+ this._data.corruptDate = new Date();
+ }
+ }
+ }.bind(this));
+ },
+
+ /**
+ * Save data to disk.
+ *
+ * @return Promise<null>
+ */
+ save: function () {
+ return Task.spawn(function* () {
+ if (!this._data) {
+ return;
+ }
+
+ let normalized = {
+ // The version should be incremented whenever the format
+ // changes.
+ v: 1,
+ // Maps crash IDs to objects defining the crash.
+ crashes: {},
+ // Maps days since UNIX epoch to objects mapping event types to
+ // counts. This is a mirror of this._countsByDay. e.g.
+ // {
+ // 15000: {
+ // "main-crash": 2,
+ // "plugin-crash": 1
+ // }
+ // }
+ countsByDay: {},
+
+ // When the store was last corrupted.
+ corruptDate: null,
+ };
+
+ if (this._data.corruptDate) {
+ normalized.corruptDate = this._data.corruptDate.getTime();
+ }
+
+ for (let [id, crash] of this._data.crashes) {
+ let c = this._normalize(crash);
+
+ c.submissions = {};
+ for (let [submissionID, submission] of crash.submissions) {
+ c.submissions[submissionID] = this._normalize(submission);
+ }
+
+ normalized.crashes[id] = c;
+ }
+
+ for (let [day, m] of this._countsByDay) {
+ normalized.countsByDay[day] = {};
+ for (let [type, count] of m) {
+ normalized.countsByDay[day][type] = count;
+ }
+ }
+
+ let encoder = new TextEncoder();
+ let data = encoder.encode(JSON.stringify(normalized));
+ let size = yield OS.File.writeAtomic(this._storePath, data, {
+ tmpPath: this._storePath + ".tmp",
+ compression: "lz4"});
+ if (this._telemetrySizeKey) {
+ Services.telemetry.getHistogramById(this._telemetrySizeKey).add(size);
+ }
+ }.bind(this));
+ },
+
+ /**
+ * Normalize an object into one fit for serialization.
+ *
+ * This function along with _denormalize() serve to hack around the
+ * default handling of Date JSON serialization because Date serialization
+ * is undefined by JSON.
+ *
+ * Fields ending with "Date" are assumed to contain Date instances.
+ * We convert these to milliseconds since epoch on output and back to
+ * Date on input.
+ */
+ _normalize: function (o) {
+ let normalized = {};
+
+ for (let k in o) {
+ let v = o[k];
+ if (v && k.endsWith("Date")) {
+ normalized[k] = v.getTime();
+ } else {
+ normalized[k] = v;
+ }
+ }
+
+ return normalized;
+ },
+
+ /**
+ * Convert a serialized object back to its native form.
+ */
+ _denormalize: function (o) {
+ let n = {};
+
+ for (let k in o) {
+ let v = o[k];
+ if (v && k.endsWith("Date")) {
+ n[k] = new Date(parseInt(v, 10));
+ } else {
+ n[k] = v;
+ }
+ }
+
+ return n;
+ },
+
+ /**
+ * Prune old crash data.
+ *
+ * Crashes without recent activity are pruned from the store so the
+ * size of the store is not unbounded. If there is activity on a crash,
+ * that activity will keep the crash and all its data around for longer.
+ *
+ * @param date
+ * (Date) The cutoff at which data will be pruned. If an entry
+ * doesn't have data newer than this, it will be pruned.
+ */
+ pruneOldCrashes: function (date) {
+ for (let crash of this.crashes) {
+ let newest = crash.newestDate;
+ if (!newest || newest.getTime() < date.getTime()) {
+ this._data.crashes.delete(crash.id);
+ }
+ }
+ },
+
+ /**
+ * Date the store was last corrupted and required a reset.
+ *
+ * May be null (no corruption has ever occurred) or a Date instance.
+ */
+ get corruptDate() {
+ return this._data.corruptDate;
+ },
+
+ /**
+ * The number of distinct crashes tracked.
+ */
+ get crashesCount() {
+ return this._data.crashes.size;
+ },
+
+ /**
+ * All crashes tracked.
+ *
+ * This is an array of CrashRecord.
+ */
+ get crashes() {
+ let crashes = [];
+ for (let [, crash] of this._data.crashes) {
+ crashes.push(new CrashRecord(crash));
+ }
+
+ return crashes;
+ },
+
+ /**
+ * Obtain a particular crash from its ID.
+ *
+ * A CrashRecord will be returned if the crash exists. null will be returned
+ * if the crash is unknown.
+ */
+ getCrash: function (id) {
+ for (let crash of this.crashes) {
+ if (crash.id == id) {
+ return crash;
+ }
+ }
+
+ return null;
+ },
+
+ _ensureCountsForDay: function (day) {
+ if (!this._countsByDay.has(day)) {
+ this._countsByDay.set(day, new Map());
+ }
+ },
+
+ /**
+ * Ensure the crash record is present in storage.
+ *
+ * Returns the crash record if we're allowed to store it or null
+ * if we've hit the high water mark.
+ *
+ * @param processType
+ * (string) One of the PROCESS_TYPE constants.
+ * @param crashType
+ * (string) One of the CRASH_TYPE constants.
+ * @param id
+ * (string) The crash ID.
+ * @param date
+ * (Date) When this crash occurred.
+ * @param metadata
+ * (dictionary) Crash metadata, may be empty.
+ *
+ * @return null | object crash record
+ */
+ _ensureCrashRecord: function (processType, crashType, id, date, metadata) {
+ if (!id) {
+ // Crashes are keyed on ID, so it's not really helpful to store crashes
+ // without IDs.
+ return null;
+ }
+
+ let type = processType + "-" + crashType;
+
+ if (!this._data.crashes.has(id)) {
+ let day = dateToDays(date);
+ this._ensureCountsForDay(day);
+
+ let count = (this._countsByDay.get(day).get(type) || 0) + 1;
+ this._countsByDay.get(day).set(type, count);
+
+ if (count > this.HIGH_WATER_DAILY_THRESHOLD &&
+ processType != CrashManager.prototype.PROCESS_TYPE_MAIN) {
+ return null;
+ }
+
+ // If we have an OOM size, count the crash as an OOM in addition to
+ // being a main process crash.
+ if (metadata && metadata.OOMAllocationSize) {
+ let oomType = type + "-oom";
+ let oomCount = (this._countsByDay.get(day).get(oomType) || 0) + 1;
+ this._countsByDay.get(day).set(oomType, oomCount);
+ }
+
+ this._data.crashes.set(id, {
+ id: id,
+ remoteID: null,
+ type: type,
+ crashDate: date,
+ submissions: new Map(),
+ classifications: [],
+ metadata: metadata,
+ });
+ }
+
+ let crash = this._data.crashes.get(id);
+ crash.type = type;
+ crash.crashDate = date;
+
+ return crash;
+ },
+
+ /**
+ * Record the occurrence of a crash.
+ *
+ * @param processType (string) One of the PROCESS_TYPE constants.
+ * @param crashType (string) One of the CRASH_TYPE constants.
+ * @param id (string) Crash ID. Likely a UUID.
+ * @param date (Date) When the crash occurred.
+ * @param metadata (dictionary) Crash metadata, may be empty.
+ *
+ * @return boolean True if the crash was recorded and false if not.
+ */
+ addCrash: function (processType, crashType, id, date, metadata) {
+ return !!this._ensureCrashRecord(processType, crashType, id, date, metadata);
+ },
+
+ /**
+ * @return boolean True if the remote ID was recorded and false if not.
+ */
+ setRemoteCrashID: function (crashID, remoteID) {
+ let crash = this._data.crashes.get(crashID);
+ if (!crash || !remoteID) {
+ return false;
+ }
+
+ crash.remoteID = remoteID;
+ return true;
+ },
+
+ getCrashesOfType: function (processType, crashType) {
+ let crashes = [];
+ for (let crash of this.crashes) {
+ if (crash.isOfType(processType, crashType)) {
+ crashes.push(crash);
+ }
+ }
+
+ return crashes;
+ },
+
+ /**
+ * Ensure the submission record is present in storage.
+ * @returns [submission, crash]
+ */
+ _ensureSubmissionRecord: function (crashID, submissionID) {
+ let crash = this._data.crashes.get(crashID);
+ if (!crash || !submissionID) {
+ return null;
+ }
+
+ if (!crash.submissions.has(submissionID)) {
+ crash.submissions.set(submissionID, {
+ requestDate: null,
+ responseDate: null,
+ result: null,
+ });
+ }
+
+ return [crash.submissions.get(submissionID), crash];
+ },
+
+ /**
+ * @return boolean True if the attempt was recorded.
+ */
+ addSubmissionAttempt: function (crashID, submissionID, date) {
+ let [submission, crash] =
+ this._ensureSubmissionRecord(crashID, submissionID);
+ if (!submission) {
+ return false;
+ }
+
+ submission.requestDate = date;
+ Services.telemetry.getKeyedHistogramById("PROCESS_CRASH_SUBMIT_ATTEMPT")
+ .add(crash.type, 1);
+ return true;
+ },
+
+ /**
+ * @return boolean True if the response was recorded.
+ */
+ addSubmissionResult: function (crashID, submissionID, date, result) {
+ let crash = this._data.crashes.get(crashID);
+ if (!crash || !submissionID) {
+ return false;
+ }
+ let submission = crash.submissions.get(submissionID);
+ if (!submission) {
+ return false;
+ }
+
+ submission.responseDate = date;
+ submission.result = result;
+ Services.telemetry.getKeyedHistogramById("PROCESS_CRASH_SUBMIT_SUCCESS")
+ .add(crash.type, result == "ok");
+ return true;
+ },
+
+ /**
+ * @return boolean True if the classifications were set.
+ */
+ setCrashClassifications: function (crashID, classifications) {
+ let crash = this._data.crashes.get(crashID);
+ if (!crash) {
+ return false;
+ }
+
+ crash.classifications = classifications;
+ return true;
+ },
+});
+
+/**
+ * Represents an individual crash with metadata.
+ *
+ * This is a wrapper around the low-level anonymous JS objects that define
+ * crashes. It exposes a consistent and helpful API.
+ *
+ * Instances of this type should only be constructured inside this module,
+ * not externally. The constructor is not considered a public API.
+ *
+ * @param o (object)
+ * The crash's entry from the CrashStore.
+ */
+function CrashRecord(o) {
+ this._o = o;
+}
+
+CrashRecord.prototype = Object.freeze({
+ get id() {
+ return this._o.id;
+ },
+
+ get remoteID() {
+ return this._o.remoteID;
+ },
+
+ get crashDate() {
+ return this._o.crashDate;
+ },
+
+ /**
+ * Obtain the newest date in this record.
+ *
+ * This is a convenience getter. The returned value is used to determine when
+ * to expire a record.
+ */
+ get newestDate() {
+ // We currently only have 1 date, so this is easy.
+ return this._o.crashDate;
+ },
+
+ get oldestDate() {
+ return this._o.crashDate;
+ },
+
+ get type() {
+ return this._o.type;
+ },
+
+ isOfType: function (processType, crashType) {
+ return processType + "-" + crashType == this.type;
+ },
+
+ get submissions() {
+ return this._o.submissions;
+ },
+
+ get classifications() {
+ return this._o.classifications;
+ },
+
+ get metadata() {
+ return this._o.metadata;
+ },
+});
+
+/**
+ * Obtain the global CrashManager instance used by the running application.
+ *
+ * CrashManager is likely only ever instantiated once per application lifetime.
+ * The main reason it's implemented as a reusable type is to facilitate testing.
+ */
+XPCOMUtils.defineLazyGetter(this.CrashManager, "Singleton", function () {
+ if (gCrashManager) {
+ return gCrashManager;
+ }
+
+ let crPath = OS.Path.join(OS.Constants.Path.userApplicationDataDir,
+ "Crash Reports");
+ let storePath = OS.Path.join(OS.Constants.Path.profileDir, "crashes");
+
+ gCrashManager = new CrashManager({
+ pendingDumpsDir: OS.Path.join(crPath, "pending"),
+ submittedDumpsDir: OS.Path.join(crPath, "submitted"),
+ eventsDirs: [OS.Path.join(crPath, "events"), OS.Path.join(storePath, "events")],
+ storeDir: storePath,
+ telemetryStoreSizeKey: "CRASH_STORE_COMPRESSED_BYTES",
+ });
+
+ // Automatically aggregate event files shortly after startup. This
+ // ensures it happens with some frequency.
+ //
+ // There are performance considerations here. While this is doing
+ // work and could negatively impact performance, the amount of work
+ // is kept small per run by periodically aggregating event files.
+ // Furthermore, well-behaving installs should not have much work
+ // here to do. If there is a lot of work, that install has bigger
+ // issues beyond reduced performance near startup.
+ gCrashManager.scheduleMaintenance(AGGREGATE_STARTUP_DELAY_MS);
+
+ return gCrashManager;
+});