diff options
author | Matt A. Tobin <email@mattatobin.com> | 2018-02-02 03:32:58 -0500 |
---|---|---|
committer | Matt A. Tobin <email@mattatobin.com> | 2018-02-02 03:32:58 -0500 |
commit | e72ef92b5bdc43cd2584198e2e54e951b70299e8 (patch) | |
tree | 01ceb4a897c33eca9e7ccf2bc3aefbe530169fe5 /application/basilisk/experiments | |
parent | 0d19b77d3eaa5b8d837bf52c19759e68e42a1c4c (diff) | |
download | UXP-e72ef92b5bdc43cd2584198e2e54e951b70299e8.tar UXP-e72ef92b5bdc43cd2584198e2e54e951b70299e8.tar.gz UXP-e72ef92b5bdc43cd2584198e2e54e951b70299e8.tar.lz UXP-e72ef92b5bdc43cd2584198e2e54e951b70299e8.tar.xz UXP-e72ef92b5bdc43cd2584198e2e54e951b70299e8.zip |
Add Basilisk
Diffstat (limited to 'application/basilisk/experiments')
29 files changed, 6774 insertions, 0 deletions
diff --git a/application/basilisk/experiments/.eslintrc.js b/application/basilisk/experiments/.eslintrc.js new file mode 100644 index 000000000..1f6b11d67 --- /dev/null +++ b/application/basilisk/experiments/.eslintrc.js @@ -0,0 +1,11 @@ +"use strict"; + +module.exports = { + "rules": { + "no-unused-vars": ["error", { + "vars": "all", + "varsIgnorePattern": "^(Cc|Ci|Cr|Cu|EXPORTED_SYMBOLS)$", + "args": "none" + }] + } +}; diff --git a/application/basilisk/experiments/Experiments.jsm b/application/basilisk/experiments/Experiments.jsm new file mode 100644 index 000000000..5a77f0997 --- /dev/null +++ b/application/basilisk/experiments/Experiments.jsm @@ -0,0 +1,2340 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +this.EXPORTED_SYMBOLS = [ + "Experiments", +]; + +const {classes: Cc, interfaces: Ci, utils: Cu} = Components; + +Cu.import("resource://gre/modules/XPCOMUtils.jsm"); +Cu.import("resource://gre/modules/Services.jsm"); +Cu.import("resource://gre/modules/Task.jsm"); +Cu.import("resource://gre/modules/Promise.jsm"); +Cu.import("resource://gre/modules/osfile.jsm"); +Cu.import("resource://gre/modules/Log.jsm"); +Cu.import("resource://gre/modules/Preferences.jsm"); +Cu.import("resource://gre/modules/AsyncShutdown.jsm"); + +XPCOMUtils.defineLazyModuleGetter(this, "UpdateUtils", + "resource://gre/modules/UpdateUtils.jsm"); +XPCOMUtils.defineLazyModuleGetter(this, "AddonManager", + "resource://gre/modules/AddonManager.jsm"); +XPCOMUtils.defineLazyModuleGetter(this, "AddonManagerPrivate", + "resource://gre/modules/AddonManager.jsm"); +XPCOMUtils.defineLazyModuleGetter(this, "TelemetryEnvironment", + "resource://gre/modules/TelemetryEnvironment.jsm"); +XPCOMUtils.defineLazyModuleGetter(this, "TelemetryLog", + "resource://gre/modules/TelemetryLog.jsm"); +XPCOMUtils.defineLazyModuleGetter(this, "TelemetryUtils", + "resource://gre/modules/TelemetryUtils.jsm"); +XPCOMUtils.defineLazyModuleGetter(this, "CommonUtils", + "resource://services-common/utils.js"); + +XPCOMUtils.defineLazyServiceGetter(this, "gCrashReporter", + "@mozilla.org/xre/app-info;1", + "nsICrashReporter"); + +const FILE_CACHE = "experiments.json"; +const EXPERIMENTS_CHANGED_TOPIC = "experiments-changed"; +const MANIFEST_VERSION = 1; +const CACHE_VERSION = 1; + +const KEEP_HISTORY_N_DAYS = 180; + +const PREF_BRANCH = "experiments."; +const PREF_ENABLED = "enabled"; // experiments.enabled +const PREF_ACTIVE_EXPERIMENT = "activeExperiment"; // whether we have an active experiment +const PREF_LOGGING = "logging"; +const PREF_LOGGING_LEVEL = PREF_LOGGING + ".level"; // experiments.logging.level +const PREF_LOGGING_DUMP = PREF_LOGGING + ".dump"; // experiments.logging.dump +const PREF_MANIFEST_URI = "manifest.uri"; // experiments.logging.manifest.uri +const PREF_FORCE_SAMPLE = "force-sample-value"; // experiments.force-sample-value + +const PREF_BRANCH_TELEMETRY = "toolkit.telemetry."; +const PREF_TELEMETRY_ENABLED = "enabled"; + +const URI_EXTENSION_STRINGS = "chrome://mozapps/locale/extensions/extensions.properties"; +const STRING_TYPE_NAME = "type.%ID%.name"; + +const CACHE_WRITE_RETRY_DELAY_SEC = 60 * 3; +const MANIFEST_FETCH_TIMEOUT_MSEC = 60 * 3 * 1000; // 3 minutes + +const TELEMETRY_LOG = { + // log(key, [kind, experimentId, details]) + ACTIVATION_KEY: "EXPERIMENT_ACTIVATION", + ACTIVATION: { + // Successfully activated. + ACTIVATED: "ACTIVATED", + // Failed to install the add-on. + INSTALL_FAILURE: "INSTALL_FAILURE", + // Experiment does not meet activation requirements. Details will + // be provided. + REJECTED: "REJECTED", + }, + + // log(key, [kind, experimentId, optionalDetails...]) + TERMINATION_KEY: "EXPERIMENT_TERMINATION", + TERMINATION: { + // The Experiments service was disabled. + SERVICE_DISABLED: "SERVICE_DISABLED", + // Add-on uninstalled. + ADDON_UNINSTALLED: "ADDON_UNINSTALLED", + // The experiment disabled itself. + FROM_API: "FROM_API", + // The experiment expired (e.g. by exceeding the end date). + EXPIRED: "EXPIRED", + // Disabled after re-evaluating conditions. If this is specified, + // details will be provided. + RECHECK: "RECHECK", + }, +}; +XPCOMUtils.defineConstant(this, "TELEMETRY_LOG", TELEMETRY_LOG); + +const gPrefs = new Preferences(PREF_BRANCH); +const gPrefsTelemetry = new Preferences(PREF_BRANCH_TELEMETRY); +var gExperimentsEnabled = false; +var gAddonProvider = null; +var gExperiments = null; +var gLogAppenderDump = null; +var gPolicyCounter = 0; +var gExperimentsCounter = 0; +var gExperimentEntryCounter = 0; +var gPreviousProviderCounter = 0; + +// Tracks active AddonInstall we know about so we can deny external +// installs. +var gActiveInstallURLs = new Set(); + +// Tracks add-on IDs that are being uninstalled by us. This allows us +// to differentiate between expected uninstalled and user-driven uninstalls. +var gActiveUninstallAddonIDs = new Set(); + +var gLogger; +var gLogDumping = false; + +function configureLogging() { + if (!gLogger) { + gLogger = Log.repository.getLogger("Browser.Experiments"); + gLogger.addAppender(new Log.ConsoleAppender(new Log.BasicFormatter())); + } + gLogger.level = gPrefs.get(PREF_LOGGING_LEVEL, Log.Level.Warn); + + let logDumping = gPrefs.get(PREF_LOGGING_DUMP, false); + if (logDumping != gLogDumping) { + if (logDumping) { + gLogAppenderDump = new Log.DumpAppender(new Log.BasicFormatter()); + gLogger.addAppender(gLogAppenderDump); + } else { + gLogger.removeAppender(gLogAppenderDump); + gLogAppenderDump = null; + } + gLogDumping = logDumping; + } +} + +// Loads a JSON file using OS.file. file is a string representing the path +// of the file to be read, options contains additional options to pass to +// OS.File.read. +// Returns a Promise resolved with the json payload or rejected with +// OS.File.Error or JSON.parse() errors. +function loadJSONAsync(file, options) { + return Task.spawn(function*() { + let rawData = yield OS.File.read(file, options); + // Read json file into a string + let data; + try { + // Obtain a converter to read from a UTF-8 encoded input stream. + let converter = new TextDecoder(); + data = JSON.parse(converter.decode(rawData)); + } catch (ex) { + gLogger.error("Experiments: Could not parse JSON: " + file + " " + ex); + throw ex; + } + return data; + }); +} + +// Returns a promise that is resolved with the AddonInstall for that URL. +function addonInstallForURL(url, hash) { + return AddonManager.getInstallForURL(url, null, "application/x-xpinstall", hash); +} + +// Returns a promise that is resolved with an Array<Addon> of the installed +// experiment addons. +function installedExperimentAddons() { + return AddonManager.getAddonsByTypes(["experiment"]).then(addons => { + return addons.filter(a => !a.appDisabled); + }); +} + +// Takes an Array<Addon> and returns a promise that is resolved when the +// addons are uninstalled. +function uninstallAddons(addons) { + let ids = new Set(addons.map(addon => addon.id)); + let deferred = Promise.defer(); + + let listener = {}; + listener.onUninstalled = addon => { + if (!ids.has(addon.id)) { + return; + } + + ids.delete(addon.id); + if (ids.size == 0) { + AddonManager.removeAddonListener(listener); + deferred.resolve(); + } + }; + + AddonManager.addAddonListener(listener); + + for (let addon of addons) { + // Disabling the add-on before uninstalling is necessary to cause tests to + // pass. This might be indicative of a bug in XPIProvider. + // TODO follow up in bug 992396. + addon.userDisabled = true; + addon.uninstall(); + } + + return deferred.promise; +} + +/** + * The experiments module. + */ + +var Experiments = { + /** + * Provides access to the global `Experiments.Experiments` instance. + */ + instance() { + if (!gExperiments) { + gExperiments = new Experiments.Experiments(); + } + + return gExperiments; + }, +}; + +/* + * The policy object allows us to inject fake enviroment data from the + * outside by monkey-patching. + */ + +Experiments.Policy = function() { + this._log = Log.repository.getLoggerWithMessagePrefix( + "Browser.Experiments.Policy", + "Policy #" + gPolicyCounter++ + "::"); + + // Set to true to ignore hash verification on downloaded XPIs. This should + // not be used outside of testing. + this.ignoreHashes = false; +}; + +Experiments.Policy.prototype = { + now() { + return new Date(); + }, + + random() { + let pref = gPrefs.get(PREF_FORCE_SAMPLE); + if (pref !== undefined) { + let val = Number.parseFloat(pref); + this._log.debug("random sample forced: " + val); + if (isNaN(val) || val < 0) { + return 0; + } + if (val > 1) { + return 1; + } + return val; + } + return Math.random(); + }, + + futureDate(offset) { + return new Date(this.now().getTime() + offset); + }, + + oneshotTimer(callback, timeout, thisObj, name) { + return CommonUtils.namedTimer(callback, timeout, thisObj, name); + }, + + updatechannel() { + return UpdateUtils.UpdateChannel; + }, + + locale() { + let chrome = Cc["@mozilla.org/chrome/chrome-registry;1"].getService(Ci.nsIXULChromeRegistry); + return chrome.getSelectedLocale("global"); + }, + + /** + * For testing a race condition, one of the tests delays the callback of + * writing the cache by replacing this policy function. + */ + delayCacheWrite(promise) { + return promise; + }, +}; + +function AlreadyShutdownError(message = "already shut down") { + Error.call(this, message); + let error = new Error(); + this.name = "AlreadyShutdownError"; + this.message = message; + this.stack = error.stack; +} +AlreadyShutdownError.prototype = Object.create(Error.prototype); +AlreadyShutdownError.prototype.constructor = AlreadyShutdownError; + +function CacheWriteError(message = "Error writing cache file") { + Error.call(this, message); + let error = new Error(); + this.name = "CacheWriteError"; + this.message = message; + this.stack = error.stack; +} +CacheWriteError.prototype = Object.create(Error.prototype); +CacheWriteError.prototype.constructor = CacheWriteError; + +/** + * Manages the experiments and provides an interface to control them. + */ + +Experiments.Experiments = function(policy = new Experiments.Policy()) { + let log = Log.repository.getLoggerWithMessagePrefix( + "Browser.Experiments.Experiments", + "Experiments #" + gExperimentsCounter++ + "::"); + + // At the time of this writing, Experiments.jsm has severe + // crashes. For forensics purposes, keep the last few log + // messages in memory and upload them in case of crash. + this._forensicsLogs = []; + this._forensicsLogs.length = 30; + this._log = Object.create(log); + this._log.log = (level, string, params) => { + this._addToForensicsLog("Experiments", string); + log.log(level, string, params); + }; + + this._log.trace("constructor"); + + // Capture the latest error, for forensics purposes. + this._latestError = null; + + + this._policy = policy; + + // This is a Map of (string -> ExperimentEntry), keyed with the experiment id. + // It holds both the current experiments and history. + // Map() preserves insertion order, which means we preserve the manifest order. + // This is null until we've successfully completed loading the cache from + // disk the first time. + this._experiments = null; + this._refresh = false; + this._terminateReason = null; // or TELEMETRY_LOG.TERMINATION.... + this._dirty = false; + + // Loading the cache happens once asynchronously on startup + this._loadTask = null; + + // The _main task handles all other actions: + // * refreshing the manifest off the network (if _refresh) + // * disabling/enabling experiments + // * saving the cache (if _dirty) + this._mainTask = null; + + // Timer for re-evaluating experiment status. + this._timer = null; + + this._shutdown = false; + this._networkRequest = null; + + // We need to tell when we first evaluated the experiments to fire an + // experiments-changed notification when we only loaded completed experiments. + this._firstEvaluate = true; + + this.init(); +}; + +Experiments.Experiments.prototype = { + QueryInterface: XPCOMUtils.generateQI([Ci.nsITimerCallback, Ci.nsIObserver]), + + /** + * `true` if the experiments manager is currently setup (has been fully initialized + * and not uninitialized yet). + */ + get isReady() { + return !this._shutdown; + }, + + init() { + this._shutdown = false; + configureLogging(); + + gExperimentsEnabled = gPrefs.get(PREF_ENABLED, false) && TelemetryUtils.isTelemetryEnabled; + this._log.trace("enabled=" + gExperimentsEnabled + ", " + this.enabled); + + gPrefs.observe(PREF_LOGGING, configureLogging); + gPrefs.observe(PREF_MANIFEST_URI, this.updateManifest, this); + gPrefs.observe(PREF_ENABLED, this._toggleExperimentsEnabled, this); + + gPrefsTelemetry.observe(PREF_TELEMETRY_ENABLED, this._telemetryStatusChanged, this); + + AddonManager.shutdown.addBlocker("Experiments.jsm shutdown", + this.uninit.bind(this), + this._getState.bind(this) + ); + + this._registerWithAddonManager(); + + this._loadTask = this._loadFromCache(); + + return this._loadTask.then( + () => { + this._log.trace("_loadTask finished ok"); + this._loadTask = null; + return this._run(); + }, + (e) => { + this._log.error("_loadFromCache caught error: " + e); + this._latestError = e; + throw e; + } + ); + }, + + /** + * Uninitialize this instance. + * + * This function is susceptible to race conditions. If it is called multiple + * times before the previous uninit() has completed or if it is called while + * an init() operation is being performed, the object may get in bad state + * and/or deadlock could occur. + * + * @return Promise<> + * The promise is fulfilled when all pending tasks are finished. + */ + uninit: Task.async(function* () { + this._log.trace("uninit: started"); + yield this._loadTask; + this._log.trace("uninit: finished with _loadTask"); + + if (!this._shutdown) { + this._log.trace("uninit: no previous shutdown"); + this._unregisterWithAddonManager(); + + gPrefs.ignore(PREF_LOGGING, configureLogging); + gPrefs.ignore(PREF_MANIFEST_URI, this.updateManifest, this); + gPrefs.ignore(PREF_ENABLED, this._toggleExperimentsEnabled, this); + + gPrefsTelemetry.ignore(PREF_TELEMETRY_ENABLED, this._telemetryStatusChanged, this); + + if (this._timer) { + this._timer.clear(); + } + } + + this._shutdown = true; + if (this._mainTask) { + if (this._networkRequest) { + try { + this._log.trace("Aborting pending network request: " + this._networkRequest); + this._networkRequest.abort(); + } catch (e) { + // pass + } + } + try { + this._log.trace("uninit: waiting on _mainTask"); + yield this._mainTask; + } catch (e) { + // We error out of tasks after shutdown via this exception. + this._log.trace(`uninit: caught error - ${e}`); + if (!(e instanceof AlreadyShutdownError)) { + this._latestError = e; + throw e; + } + } + } + + this._log.info("Completed uninitialization."); + }), + + // Return state information, for debugging purposes. + _getState() { + let activeExperiment = this._getActiveExperiment(); + let state = { + isShutdown: this._shutdown, + isEnabled: gExperimentsEnabled, + isRefresh: this._refresh, + isDirty: this._dirty, + isFirstEvaluate: this._firstEvaluate, + hasLoadTask: !!this._loadTask, + hasMainTask: !!this._mainTask, + hasTimer: !!this._hasTimer, + hasAddonProvider: !!gAddonProvider, + latestLogs: this._forensicsLogs, + experiments: this._experiments ? [...this._experiments.keys()] : null, + terminateReason: this._terminateReason, + activeExperiment: activeExperiment ? activeExperiment.id : null, + }; + if (this._latestError) { + if (typeof this._latestError == "object") { + state.latestError = { + message: this._latestError.message, + stack: this._latestError.stack + }; + } else { + state.latestError = "" + this._latestError; + } + } + return state; + }, + + _addToForensicsLog(what, string) { + this._forensicsLogs.shift(); + let timeInSec = Math.floor(Services.telemetry.msSinceProcessStart() / 1000); + this._forensicsLogs.push(`${timeInSec}: ${what} - ${string}`); + }, + + _registerWithAddonManager(previousExperimentsProvider) { + this._log.trace("Registering instance with Addon Manager."); + + AddonManager.addAddonListener(this); + AddonManager.addInstallListener(this); + + if (!gAddonProvider) { + // The properties of this AddonType should be kept in sync with the + // experiment AddonType registered in XPIProvider. + this._log.trace("Registering previous experiment add-on provider."); + gAddonProvider = previousExperimentsProvider || new Experiments.PreviousExperimentProvider(this); + AddonManagerPrivate.registerProvider(gAddonProvider, [ + new AddonManagerPrivate.AddonType("experiment", + URI_EXTENSION_STRINGS, + STRING_TYPE_NAME, + AddonManager.VIEW_TYPE_LIST, + 11000, + AddonManager.TYPE_UI_HIDE_EMPTY), + ]); + } + + }, + + _unregisterWithAddonManager() { + this._log.trace("Unregistering instance with Addon Manager."); + + this._log.trace("Removing install listener from add-on manager."); + AddonManager.removeInstallListener(this); + this._log.trace("Removing addon listener from add-on manager."); + AddonManager.removeAddonListener(this); + this._log.trace("Finished unregistering with addon manager."); + + if (gAddonProvider) { + this._log.trace("Unregistering previous experiment add-on provider."); + AddonManagerPrivate.unregisterProvider(gAddonProvider); + gAddonProvider = null; + } + }, + + /* + * Change the PreviousExperimentsProvider that this instance uses. + * For testing only. + */ + _setPreviousExperimentsProvider(provider) { + this._unregisterWithAddonManager(); + this._registerWithAddonManager(provider); + }, + + /** + * Throws an exception if we've already shut down. + */ + _checkForShutdown() { + if (this._shutdown) { + throw new AlreadyShutdownError("uninit() already called"); + } + }, + + /** + * Whether the experiments feature is enabled. + */ + get enabled() { + return gExperimentsEnabled; + }, + + /** + * Toggle whether the experiments feature is enabled or not. + */ + set enabled(enabled) { + this._log.trace("set enabled(" + enabled + ")"); + gPrefs.set(PREF_ENABLED, enabled); + }, + + _toggleExperimentsEnabled: Task.async(function* (enabled) { + this._log.trace("_toggleExperimentsEnabled(" + enabled + ")"); + let wasEnabled = gExperimentsEnabled; + gExperimentsEnabled = enabled && TelemetryUtils.isTelemetryEnabled; + + if (wasEnabled == gExperimentsEnabled) { + return; + } + + if (gExperimentsEnabled) { + yield this.updateManifest(); + } else { + yield this.disableExperiment(TELEMETRY_LOG.TERMINATION.SERVICE_DISABLED); + if (this._timer) { + this._timer.clear(); + } + } + }), + + _telemetryStatusChanged() { + this._toggleExperimentsEnabled(gExperimentsEnabled); + }, + + /** + * Returns a promise that is resolved with an array of `ExperimentInfo` objects, + * which provide info on the currently and recently active experiments. + * The array is in chronological order. + * + * The experiment info is of the form: + * { + * id: <string>, + * name: <string>, + * description: <string>, + * active: <boolean>, + * endDate: <integer>, // epoch ms + * detailURL: <string>, + * ... // possibly extended later + * } + * + * @return Promise<Array<ExperimentInfo>> Array of experiment info objects. + */ + getExperiments() { + return Task.spawn(function*() { + yield this._loadTask; + let list = []; + + for (let [id, experiment] of this._experiments) { + if (!experiment.startDate) { + // We only collect experiments that are or were active. + continue; + } + + list.push({ + id, + name: experiment._name, + description: experiment._description, + active: experiment.enabled, + endDate: experiment.endDate.getTime(), + detailURL: experiment._homepageURL, + branch: experiment.branch, + }); + } + + // Sort chronologically, descending. + list.sort((a, b) => b.endDate - a.endDate); + return list; + }.bind(this)); + }, + + /** + * Returns the ExperimentInfo for the active experiment, or null + * if there is none. + */ + getActiveExperiment() { + let experiment = this._getActiveExperiment(); + if (!experiment) { + return null; + } + + let info = { + id: experiment.id, + name: experiment._name, + description: experiment._description, + active: experiment.enabled, + endDate: experiment.endDate.getTime(), + detailURL: experiment._homepageURL, + }; + + return info; + }, + + /** + * Experiment "branch" support. If an experiment has multiple branches, it + * can record the branch with the experiment system and it will + * automatically be included in data reporting (FHR/telemetry payloads). + */ + + /** + * Set the experiment branch for the specified experiment ID. + * @returns Promise<> + */ + setExperimentBranch: Task.async(function*(id, branchstr) { + yield this._loadTask; + let e = this._experiments.get(id); + if (!e) { + throw new Error("Experiment not found"); + } + e.branch = String(branchstr); + this._log.trace("setExperimentBranch(" + id + ", " + e.branch + ") _dirty=" + this._dirty); + this._dirty = true; + Services.obs.notifyObservers(null, EXPERIMENTS_CHANGED_TOPIC, null); + yield this._run(); + }), + /** + * Get the branch of the specified experiment. If the experiment is unknown, + * throws an error. + * + * @param id The ID of the experiment. Pass null for the currently running + * experiment. + * @returns Promise<string|null> + * @throws Error if the specified experiment ID is unknown, or if there is no + * current experiment. + */ + getExperimentBranch: Task.async(function*(id = null) { + yield this._loadTask; + let e; + if (id) { + e = this._experiments.get(id); + if (!e) { + throw new Error("Experiment not found"); + } + } else { + e = this._getActiveExperiment(); + if (e === null) { + throw new Error("No active experiment"); + } + } + return e.branch; + }), + + /** + * Determine whether another date has the same UTC day as now(). + */ + _dateIsTodayUTC(d) { + let now = this._policy.now(); + + return stripDateToMidnight(now).getTime() == stripDateToMidnight(d).getTime(); + }, + + /** + * Obtain the entry of the most recent active experiment that was active + * today. + * + * If no experiment was active today, this resolves to nothing. + * + * Assumption: Only a single experiment can be active at a time. + * + * @return Promise<object> + */ + lastActiveToday() { + return Task.spawn(function* getMostRecentActiveExperimentTask() { + let experiments = yield this.getExperiments(); + + // Assumption: Ordered chronologically, descending, with active always + // first. + for (let experiment of experiments) { + if (experiment.active) { + return experiment; + } + + if (experiment.endDate && this._dateIsTodayUTC(experiment.endDate)) { + return experiment; + } + } + return null; + }.bind(this)); + }, + + _run() { + this._log.trace("_run"); + this._checkForShutdown(); + if (!this._mainTask) { + this._mainTask = Task.spawn(function*() { + try { + yield this._main(); + } catch (e) { + // In the CacheWriteError case we want to reschedule + if (!(e instanceof CacheWriteError)) { + this._log.error("_main caught error: " + e); + return; + } + } finally { + this._mainTask = null; + } + this._log.trace("_main finished, scheduling next run"); + try { + yield this._scheduleNextRun(); + } catch (ex) { + // We error out of tasks after shutdown via this exception. + if (!(ex instanceof AlreadyShutdownError)) { + throw ex; + } + } + }.bind(this)); + } + return this._mainTask; + }, + + *_main() { + do { + this._log.trace("_main iteration"); + yield this._loadTask; + if (!gExperimentsEnabled) { + this._refresh = false; + } + + if (this._refresh) { + yield this._loadManifest(); + } + yield this._evaluateExperiments(); + if (this._dirty) { + yield this._saveToCache(); + } + // If somebody called .updateManifest() or disableExperiment() + // while we were running, go again right now. + } + while (this._refresh || this._terminateReason || this._dirty); + }, + + *_loadManifest() { + this._log.trace("_loadManifest"); + let uri = Services.urlFormatter.formatURLPref(PREF_BRANCH + PREF_MANIFEST_URI); + + this._checkForShutdown(); + + this._refresh = false; + try { + let responseText = yield this._httpGetRequest(uri); + this._log.trace("_loadManifest() - responseText=\"" + responseText + "\""); + + if (this._shutdown) { + return; + } + + let data = JSON.parse(responseText); + this._updateExperiments(data); + } catch (e) { + this._log.error("_loadManifest - failure to fetch/parse manifest (continuing anyway): " + e); + } + }, + + /** + * Fetch an updated list of experiments and trigger experiment updates. + * Do only use when experiments are enabled. + * + * @return Promise<> + * The promise is resolved when the manifest and experiment list is updated. + */ + updateManifest() { + this._log.trace("updateManifest()"); + + if (!gExperimentsEnabled) { + return Promise.reject(new Error("experiments are disabled")); + } + + if (this._shutdown) { + return Promise.reject(Error("uninit() alrady called")); + } + + this._refresh = true; + return this._run(); + }, + + notify(timer) { + this._log.trace("notify()"); + this._checkForShutdown(); + return this._run(); + }, + + // START OF ADD-ON LISTENERS + + onUninstalled(addon) { + this._log.trace("onUninstalled() - addon id: " + addon.id); + if (gActiveUninstallAddonIDs.has(addon.id)) { + this._log.trace("matches pending uninstall"); + return; + } + let activeExperiment = this._getActiveExperiment(); + if (!activeExperiment || activeExperiment._addonId != addon.id) { + return; + } + + this.disableExperiment(TELEMETRY_LOG.TERMINATION.ADDON_UNINSTALLED); + }, + + /** + * @returns {Boolean} returns false when we cancel the install. + */ + onInstallStarted(install) { + if (install.addon.type != "experiment") { + return true; + } + + this._log.trace("onInstallStarted() - " + install.addon.id); + if (install.addon.appDisabled) { + // This is a PreviousExperiment + return true; + } + + // We want to be in control of all experiment add-ons: reject installs + // for add-ons that we don't know about. + + // We have a race condition of sorts to worry about here. We have 2 + // onInstallStarted listeners. This one (the global one) and the one + // created as part of ExperimentEntry._installAddon. Because of the order + // they are registered in, this one likely executes first. Unfortunately, + // this means that the add-on ID is not yet set on the ExperimentEntry. + // So, we can't just look at this._trackedAddonIds because the new experiment + // will have its add-on ID set to null. We work around this by storing a + // identifying field - the source URL of the install - in a module-level + // variable (so multiple Experiments instances doesn't cancel each other + // out). + + if (this._trackedAddonIds.has(install.addon.id)) { + this._log.info("onInstallStarted allowing install because add-on ID " + + "tracked by us."); + return true; + } + + if (gActiveInstallURLs.has(install.sourceURI.spec)) { + this._log.info("onInstallStarted allowing install because install " + + "tracked by us."); + return true; + } + + this._log.warn("onInstallStarted cancelling install of unknown " + + "experiment add-on: " + install.addon.id); + return false; + }, + + // END OF ADD-ON LISTENERS. + + _getExperimentByAddonId(addonId) { + for (let [, entry] of this._experiments) { + if (entry._addonId === addonId) { + return entry; + } + } + + return null; + }, + + /* + * Helper function to make HTTP GET requests. Returns a promise that is resolved with + * the responseText when the request is complete. + */ + _httpGetRequest(url) { + this._log.trace("httpGetRequest(" + url + ")"); + let xhr = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"].createInstance(Ci.nsIXMLHttpRequest); + + this._networkRequest = xhr; + let deferred = Promise.defer(); + + let log = this._log; + let errorhandler = (evt) => { + log.error("httpGetRequest::onError() - Error making request to " + url + ": " + evt.type); + deferred.reject(new Error("Experiments - XHR error for " + url + " - " + evt.type)); + this._networkRequest = null; + }; + xhr.onerror = errorhandler; + xhr.ontimeout = errorhandler; + xhr.onabort = errorhandler; + + xhr.onload = (event) => { + if (xhr.status !== 200 && xhr.state !== 0) { + log.error("httpGetRequest::onLoad() - Request to " + url + " returned status " + xhr.status); + deferred.reject(new Error("Experiments - XHR status for " + url + " is " + xhr.status)); + this._networkRequest = null; + return; + } + + deferred.resolve(xhr.responseText); + this._networkRequest = null; + }; + + try { + xhr.open("GET", url); + + if (xhr.channel instanceof Ci.nsISupportsPriority) { + xhr.channel.priority = Ci.nsISupportsPriority.PRIORITY_LOWEST; + } + + xhr.timeout = MANIFEST_FETCH_TIMEOUT_MSEC; + xhr.send(null); + } catch (e) { + this._log.error("httpGetRequest() - Error opening request to " + url + ": " + e); + return Promise.reject(new Error("Experiments - Error opening XHR for " + url)); + } + return deferred.promise; + }, + + /* + * Path of the cache file we use in the profile. + */ + get _cacheFilePath() { + return OS.Path.join(OS.Constants.Path.profileDir, FILE_CACHE); + }, + + /* + * Part of the main task to save the cache to disk, called from _main. + */ + *_saveToCache() { + this._log.trace("_saveToCache"); + let path = this._cacheFilePath; + this._dirty = false; + try { + let textData = JSON.stringify({ + version: CACHE_VERSION, + data: [...this._experiments.values()].map(e => e.toJSON()), + }); + + let encoder = new TextEncoder(); + let data = encoder.encode(textData); + let options = { tmpPath: path + ".tmp", compression: "lz4" }; + yield this._policy.delayCacheWrite(OS.File.writeAtomic(path, data, options)); + } catch (e) { + // We failed to write the cache, it's still dirty. + this._dirty = true; + this._log.error("_saveToCache failed and caught error: " + e); + throw new CacheWriteError(); + } + + this._log.debug("_saveToCache saved to " + path); + }, + + /* + * Task function, load the cached experiments manifest file from disk. + */ + _loadFromCache: Task.async(function* () { + this._log.trace("_loadFromCache"); + let path = this._cacheFilePath; + try { + let result = yield loadJSONAsync(path, { compression: "lz4" }); + this._populateFromCache(result); + } catch (e) { + if (e instanceof OS.File.Error && e.becauseNoSuchFile) { + // No cached manifest yet. + this._experiments = new Map(); + } else { + throw e; + } + } + }), + + _populateFromCache(data) { + this._log.trace("populateFromCache() - data: " + JSON.stringify(data)); + + // If the user has a newer cache version than we can understand, we fail + // hard; no experiments should be active in this older client. + if (CACHE_VERSION !== data.version) { + throw new Error("Experiments::_populateFromCache() - invalid cache version"); + } + + let experiments = new Map(); + for (let item of data.data) { + let entry = new Experiments.ExperimentEntry(this._policy); + if (!entry.initFromCacheData(item)) { + continue; + } + + // Discard old experiments if they ended more than 180 days ago. + if (entry.shouldDiscard()) { + // We discarded an experiment, the cache needs to be updated. + this._dirty = true; + continue; + } + + experiments.set(entry.id, entry); + } + + this._experiments = experiments; + }, + + /* + * Update the experiment entries from the experiments + * array in the manifest + */ + _updateExperiments(manifestObject) { + this._log.trace("_updateExperiments() - experiments: " + JSON.stringify(manifestObject)); + + if (manifestObject.version !== MANIFEST_VERSION) { + this._log.warning("updateExperiments() - unsupported version " + manifestObject.version); + } + + let experiments = new Map(); // The new experiments map + + // Collect new and updated experiments. + for (let data of manifestObject.experiments) { + let entry = this._experiments.get(data.id); + + if (entry) { + if (!entry.updateFromManifestData(data)) { + this._log.error("updateExperiments() - Invalid manifest data for " + data.id); + continue; + } + } else { + entry = new Experiments.ExperimentEntry(this._policy); + if (!entry.initFromManifestData(data)) { + continue; + } + } + + if (entry.shouldDiscard()) { + continue; + } + + experiments.set(entry.id, entry); + } + + // Make sure we keep experiments that are or were running. + // We remove them after KEEP_HISTORY_N_DAYS. + for (let [id, entry] of this._experiments) { + if (experiments.has(id)) { + continue; + } + + if (!entry.startDate || entry.shouldDiscard()) { + this._log.trace("updateExperiments() - discarding entry for " + id); + continue; + } + + experiments.set(id, entry); + } + + this._experiments = experiments; + this._dirty = true; + }, + + getActiveExperimentID() { + if (!this._experiments) { + return null; + } + let e = this._getActiveExperiment(); + if (!e) { + return null; + } + return e.id; + }, + + getActiveExperimentBranch() { + if (!this._experiments) { + return null; + } + let e = this._getActiveExperiment(); + if (!e) { + return null; + } + return e.branch; + }, + + _getActiveExperiment() { + let enabled = [...this._experiments.values()].filter(experiment => experiment._enabled); + + if (enabled.length == 1) { + return enabled[0]; + } + + if (enabled.length > 1) { + this._log.error("getActiveExperimentId() - should not have more than 1 active experiment"); + throw new Error("have more than 1 active experiment"); + } + + return null; + }, + + /** + * Disables all active experiments. + * + * @return Promise<> Promise that will get resolved once the task is done or failed. + */ + disableExperiment(reason) { + if (!reason) { + throw new Error("Must specify a termination reason."); + } + + this._log.trace("disableExperiment()"); + this._terminateReason = reason; + return this._run(); + }, + + /** + * The Set of add-on IDs that we know about from manifests. + */ + get _trackedAddonIds() { + if (!this._experiments) { + return new Set(); + } + + return new Set([...this._experiments.values()].map(e => e._addonId)); + }, + + /* + * Task function to check applicability of experiments, disable the active + * experiment if needed and activate the first applicable candidate. + */ + *_evaluateExperiments() { + this._log.trace("_evaluateExperiments"); + + this._checkForShutdown(); + + // The first thing we do is reconcile our state against what's in the + // Addon Manager. It's possible that the Addon Manager knows of experiment + // add-ons that we don't. This could happen if an experiment gets installed + // when we're not listening or if there is a bug in our synchronization + // code. + // + // We have a few options of what to do with unknown experiment add-ons + // coming from the Addon Manager. Ideally, we'd convert these to + // ExperimentEntry instances and stuff them inside this._experiments. + // However, since ExperimentEntry contain lots of metadata from the + // manifest and trying to make up data could be error prone, it's safer + // to not try. Furthermore, if an experiment really did come from us, we + // should have some record of it. In the end, we decide to discard all + // knowledge for these unknown experiment add-ons. + let installedExperiments = yield installedExperimentAddons(); + let expectedAddonIds = this._trackedAddonIds; + let unknownAddons = installedExperiments.filter(a => !expectedAddonIds.has(a.id)); + if (unknownAddons.length) { + this._log.warn("_evaluateExperiments() - unknown add-ons in AddonManager: " + + unknownAddons.map(a => a.id).join(", ")); + + yield uninstallAddons(unknownAddons); + } + + let activeExperiment = this._getActiveExperiment(); + let activeChanged = false; + + if (!activeExperiment) { + // Avoid this pref staying out of sync if there were e.g. crashes. + gPrefs.set(PREF_ACTIVE_EXPERIMENT, false); + } + + // Ensure the active experiment is in the proper state. This may install, + // uninstall, upgrade, or enable the experiment add-on. What exactly is + // abstracted away from us by design. + if (activeExperiment) { + let changes; + let shouldStopResult = yield activeExperiment.shouldStop(); + if (shouldStopResult.shouldStop) { + let expireReasons = ["endTime", "maxActiveSeconds"]; + let kind, reason; + + if (expireReasons.indexOf(shouldStopResult.reason[0]) != -1) { + kind = TELEMETRY_LOG.TERMINATION.EXPIRED; + reason = null; + } else { + kind = TELEMETRY_LOG.TERMINATION.RECHECK; + reason = shouldStopResult.reason; + } + changes = yield activeExperiment.stop(kind, reason); + } else if (this._terminateReason) { + changes = yield activeExperiment.stop(this._terminateReason); + } else { + changes = yield activeExperiment.reconcileAddonState(); + } + + if (changes) { + this._dirty = true; + activeChanged = true; + } + + if (!activeExperiment._enabled) { + activeExperiment = null; + activeChanged = true; + } + } + + this._terminateReason = null; + + if (!activeExperiment && gExperimentsEnabled) { + for (let [id, experiment] of this._experiments) { + let applicable; + let reason = null; + try { + applicable = yield experiment.isApplicable(); + } catch (e) { + applicable = false; + reason = e; + } + + if (!applicable && reason && reason[0] != "was-active") { + // Report this from here to avoid over-reporting. + let data = [TELEMETRY_LOG.ACTIVATION.REJECTED, id]; + data = data.concat(reason); + const key = TELEMETRY_LOG.ACTIVATION_KEY; + TelemetryLog.log(key, data); + this._log.trace("evaluateExperiments() - added " + key + " to TelemetryLog: " + JSON.stringify(data)); + } + + if (!applicable) { + continue; + } + + this._log.debug("evaluateExperiments() - activating experiment " + id); + try { + yield experiment.start(); + activeChanged = true; + activeExperiment = experiment; + this._dirty = true; + break; + } catch (e) { + // On failure, clean up the best we can and try the next experiment. + this._log.error("evaluateExperiments() - Unable to start experiment: " + e.message); + experiment._enabled = false; + yield experiment.reconcileAddonState(); + } + } + } + + gPrefs.set(PREF_ACTIVE_EXPERIMENT, activeExperiment != null); + + if (activeChanged || this._firstEvaluate) { + Services.obs.notifyObservers(null, EXPERIMENTS_CHANGED_TOPIC, null); + this._firstEvaluate = false; + } + + if ("@mozilla.org/toolkit/crash-reporter;1" in Cc && activeExperiment) { + try { + gCrashReporter.annotateCrashReport("ActiveExperiment", activeExperiment.id); + gCrashReporter.annotateCrashReport("ActiveExperimentBranch", activeExperiment.branch); + } catch (e) { + // It's ok if crash reporting is disabled. + } + } + }, + + /* + * Schedule the soonest re-check of experiment applicability that is needed. + */ + _scheduleNextRun() { + this._checkForShutdown(); + + if (this._timer) { + this._timer.clear(); + } + + if (!gExperimentsEnabled || this._experiments.length == 0) { + return; + } + + let time = null; + let now = this._policy.now().getTime(); + if (this._dirty) { + // If we failed to write the cache, we should try again periodically + time = now + 1000 * CACHE_WRITE_RETRY_DELAY_SEC; + } + + for (let [, experiment] of this._experiments) { + let scheduleTime = experiment.getScheduleTime(); + if (scheduleTime > now) { + if (time !== null) { + time = Math.min(time, scheduleTime); + } else { + time = scheduleTime; + } + } + } + + if (time === null) { + // No schedule time found. + return; + } + + this._log.trace("scheduleExperimentEvaluation() - scheduling for " + time + ", now: " + now); + this._policy.oneshotTimer(this.notify, time - now, this, "_timer"); + }, +}; + + +/* + * Represents a single experiment. + */ + +Experiments.ExperimentEntry = function(policy) { + this._policy = policy || new Experiments.Policy(); + let log = Log.repository.getLoggerWithMessagePrefix( + "Browser.Experiments.Experiments", + "ExperimentEntry #" + gExperimentEntryCounter++ + "::"); + this._log = Object.create(log); + this._log.log = (level, string, params) => { + if (gExperiments) { + gExperiments._addToForensicsLog("ExperimentEntry", string); + } + log.log(level, string, params); + }; + + // Is the experiment supposed to be running. + this._enabled = false; + // When this experiment was started, if ever. + this._startDate = null; + // When this experiment was ended, if ever. + this._endDate = null; + // The condition data from the manifest. + this._manifestData = null; + // For an active experiment, signifies whether we need to update the xpi. + this._needsUpdate = false; + // A random sample value for comparison against the manifest conditions. + this._randomValue = null; + // When this entry was last changed for respecting history retention duration. + this._lastChangedDate = null; + // Has this experiment failed to activate before? + this._failedStart = false; + // The experiment branch + this._branch = null; + + // We grab these from the addon after download. + this._name = null; + this._description = null; + this._homepageURL = null; + this._addonId = null; +}; + +Experiments.ExperimentEntry.prototype = { + MANIFEST_REQUIRED_FIELDS: new Set([ + "id", + "xpiURL", + "xpiHash", + "startTime", + "endTime", + "maxActiveSeconds", + "appName", + "channel", + ]), + + MANIFEST_OPTIONAL_FIELDS: new Set([ + "maxStartTime", + "minVersion", + "maxVersion", + "version", + "minBuildID", + "maxBuildID", + "buildIDs", + "os", + "locale", + "sample", + "disabled", + "frozen", + "jsfilter", + ]), + + SERIALIZE_KEYS: new Set([ + "_enabled", + "_manifestData", + "_needsUpdate", + "_randomValue", + "_failedStart", + "_name", + "_description", + "_homepageURL", + "_addonId", + "_startDate", + "_endDate", + "_branch", + ]), + + DATE_KEYS: new Set([ + "_startDate", + "_endDate", + ]), + + UPGRADE_KEYS: new Map([ + ["_branch", null], + ]), + + ADDON_CHANGE_NONE: 0, + ADDON_CHANGE_INSTALL: 1, + ADDON_CHANGE_UNINSTALL: 2, + ADDON_CHANGE_ENABLE: 4, + + /* + * Initialize entry from the manifest. + * @param data The experiment data from the manifest. + * @return boolean Whether initialization succeeded. + */ + initFromManifestData(data) { + if (!this._isManifestDataValid(data)) { + return false; + } + + this._manifestData = data; + + this._randomValue = this._policy.random(); + this._lastChangedDate = this._policy.now(); + + return true; + }, + + get enabled() { + return this._enabled; + }, + + get id() { + return this._manifestData.id; + }, + + get branch() { + return this._branch; + }, + + set branch(v) { + this._branch = v; + }, + + get startDate() { + return this._startDate; + }, + + get endDate() { + if (!this._startDate) { + return null; + } + + let endTime = 0; + + if (!this._enabled) { + return this._endDate; + } + + let maxActiveMs = 1000 * this._manifestData.maxActiveSeconds; + endTime = Math.min(1000 * this._manifestData.endTime, + this._startDate.getTime() + maxActiveMs); + + return new Date(endTime); + }, + + get needsUpdate() { + return this._needsUpdate; + }, + + /* + * Initialize entry from the cache. + * @param data The entry data from the cache. + * @return boolean Whether initialization succeeded. + */ + initFromCacheData(data) { + for (let [key, dval] of this.UPGRADE_KEYS) { + if (!(key in data)) { + data[key] = dval; + } + } + + for (let key of this.SERIALIZE_KEYS) { + if (!(key in data) && !this.DATE_KEYS.has(key)) { + this._log.error("initFromCacheData() - missing required key " + key); + return false; + } + } + + if (!this._isManifestDataValid(data._manifestData)) { + return false; + } + + // Dates are restored separately from epoch ms, everything else is just + // copied in. + + this.SERIALIZE_KEYS.forEach(key => { + if (!this.DATE_KEYS.has(key)) { + this[key] = data[key]; + } + }); + + this.DATE_KEYS.forEach(key => { + if (key in data) { + let date = new Date(); + date.setTime(data[key]); + this[key] = date; + } + }); + + // In order for the experiment's data expiration mechanism to work, use the experiment's + // |_endData| as the |_lastChangedDate| (if available). + this._lastChangedDate = this._endDate ? this._endDate : this._policy.now(); + + return true; + }, + + /* + * Returns a JSON representation of this object. + */ + toJSON() { + let obj = {}; + + // Dates are serialized separately as epoch ms. + + this.SERIALIZE_KEYS.forEach(key => { + if (!this.DATE_KEYS.has(key)) { + obj[key] = this[key]; + } + }); + + this.DATE_KEYS.forEach(key => { + if (this[key]) { + obj[key] = this[key].getTime(); + } + }); + + return obj; + }, + + /* + * Update from the experiment data from the manifest. + * @param data The experiment data from the manifest. + * @return boolean Whether updating succeeded. + */ + updateFromManifestData(data) { + let old = this._manifestData; + + if (!this._isManifestDataValid(data)) { + return false; + } + + if (this._enabled) { + if (old.xpiHash !== data.xpiHash) { + // A changed hash means we need to update active experiments. + this._needsUpdate = true; + } + } else if (this._failedStart && + (old.xpiHash !== data.xpiHash) || + (old.xpiURL !== data.xpiURL)) { + // Retry installation of previously invalid experiments + // if hash or url changed. + this._failedStart = false; + } + + this._manifestData = data; + this._lastChangedDate = this._policy.now(); + + return true; + }, + + /* + * Is this experiment applicable? + * @return Promise<> Resolved if the experiment is applicable. + * If it is not applicable it is rejected with + * a Promise<string> which contains the reason. + */ + isApplicable() { + let versionCmp = Cc["@mozilla.org/xpcom/version-comparator;1"] + .getService(Ci.nsIVersionComparator); + let app = Cc["@mozilla.org/xre/app-info;1"].getService(Ci.nsIXULAppInfo); + let runtime = Cc["@mozilla.org/xre/app-info;1"] + .getService(Ci.nsIXULRuntime); + + let locale = this._policy.locale(); + let channel = this._policy.updatechannel(); + let data = this._manifestData; + + let now = this._policy.now() / 1000; // The manifest times are in seconds. + let maxActive = data.maxActiveSeconds || 0; + let startSec = (this.startDate || 0) / 1000; + + this._log.trace("isApplicable() - now=" + now + + ", randomValue=" + this._randomValue); + + // Not applicable if it already ran. + + if (!this.enabled && this._endDate) { + return Promise.reject(["was-active"]); + } + + // Define and run the condition checks. + + let simpleChecks = [ + { name: "failedStart", + condition: () => !this._failedStart }, + { name: "disabled", + condition: () => !data.disabled }, + { name: "frozen", + condition: () => !data.frozen || this._enabled }, + { name: "startTime", + condition: () => now >= data.startTime }, + { name: "endTime", + condition: () => now < data.endTime }, + { name: "maxStartTime", + condition: () => this._startDate || !data.maxStartTime || now <= data.maxStartTime }, + { name: "maxActiveSeconds", + condition: () => !this._startDate || now <= (startSec + maxActive) }, + { name: "appName", + condition: () => !data.appName || data.appName.indexOf(app.name) != -1 }, + { name: "minBuildID", + condition: () => !data.minBuildID || app.platformBuildID >= data.minBuildID }, + { name: "maxBuildID", + condition: () => !data.maxBuildID || app.platformBuildID <= data.maxBuildID }, + { name: "buildIDs", + condition: () => !data.buildIDs || data.buildIDs.indexOf(app.platformBuildID) != -1 }, + { name: "os", + condition: () => !data.os || data.os.indexOf(runtime.OS) != -1 }, + { name: "channel", + condition: () => !data.channel || data.channel.indexOf(channel) != -1 }, + { name: "locale", + condition: () => !data.locale || data.locale.indexOf(locale) != -1 }, + { name: "sample", + condition: () => data.sample === undefined || this._randomValue <= data.sample }, + { name: "version", + condition: () => !data.version || data.version.indexOf(app.version) != -1 }, + { name: "minVersion", + condition: () => !data.minVersion || versionCmp.compare(app.version, data.minVersion) >= 0 }, + { name: "maxVersion", + condition: () => !data.maxVersion || versionCmp.compare(app.version, data.maxVersion) <= 0 }, + ]; + + for (let check of simpleChecks) { + let result = check.condition(); + if (!result) { + this._log.debug("isApplicable() - id=" + + data.id + " - test '" + check.name + "' failed"); + return Promise.reject([check.name]); + } + } + + if (data.jsfilter) { + return this._runFilterFunction(data.jsfilter); + } + + return Promise.resolve(true); + }, + + /* + * Run the jsfilter function from the manifest in a sandbox and return the + * result (forced to boolean). + */ + _runFilterFunction: Task.async(function* (jsfilter) { + this._log.trace("runFilterFunction() - filter: " + jsfilter); + + let ssm = Services.scriptSecurityManager; + const nullPrincipal = ssm.createNullPrincipal({}); + let options = { + sandboxName: "telemetry experiments jsfilter sandbox", + wantComponents: false, + }; + + let sandbox = Cu.Sandbox(nullPrincipal, options); + try { + Cu.evalInSandbox(jsfilter, sandbox); + } catch (e) { + this._log.error("runFilterFunction() - failed to eval jsfilter: " + e.message); + throw ["jsfilter-evalfailed"]; + } + + let currentEnvironment = yield TelemetryEnvironment.onInitialized(); + + Object.defineProperty(sandbox, "_e", + { get: () => Cu.cloneInto(currentEnvironment, sandbox) }); + + let result = false; + try { + result = !!Cu.evalInSandbox("filter({get telemetryEnvironment() { return _e; } })", sandbox); + } catch (e) { + this._log.debug("runFilterFunction() - filter function failed: " + + e.message + ", " + e.stack); + throw ["jsfilter-threw", e.message]; + } finally { + Cu.nukeSandbox(sandbox); + } + + if (!result) { + throw ["jsfilter-false"]; + } + + return true; + }), + + /* + * Start running the experiment. + * + * @return Promise<> Resolved when the operation is complete. + */ + start: Task.async(function* () { + this._log.trace("start() for " + this.id); + + this._enabled = true; + return yield this.reconcileAddonState(); + }), + + // Async install of the addon for this experiment, part of the start task above. + _installAddon: Task.async(function* () { + let deferred = Promise.defer(); + + let hash = this._policy.ignoreHashes ? null : this._manifestData.xpiHash; + + let install = yield addonInstallForURL(this._manifestData.xpiURL, hash); + gActiveInstallURLs.add(install.sourceURI.spec); + + let failureHandler = (failureInstall, handler) => { + let message = "AddonInstall " + handler + " for " + this.id + ", state=" + + (failureInstall.state || "?") + ", error=" + failureInstall.error; + this._log.error("_installAddon() - " + message); + this._failedStart = true; + gActiveInstallURLs.delete(failureInstall.sourceURI.spec); + + TelemetryLog.log(TELEMETRY_LOG.ACTIVATION_KEY, + [TELEMETRY_LOG.ACTIVATION.INSTALL_FAILURE, this.id]); + + deferred.reject(new Error(message)); + }; + + let listener = { + _expectedID: null, + + onDownloadEnded: downloadEndedInstall => { + this._log.trace("_installAddon() - onDownloadEnded for " + this.id); + + if (downloadEndedInstall.existingAddon) { + this._log.warn("_installAddon() - onDownloadEnded, addon already installed"); + } + + if (downloadEndedInstall.addon.type !== "experiment") { + this._log.error("_installAddon() - onDownloadEnded, wrong addon type"); + downloadEndedInstall.cancel(); + } + }, + + onInstallStarted: installStartedInstall => { + this._log.trace("_installAddon() - onInstallStarted for " + this.id); + + if (installStartedInstall.existingAddon) { + this._log.warn("_installAddon() - onInstallStarted, addon already installed"); + } + + if (installStartedInstall.addon.type !== "experiment") { + this._log.error("_installAddon() - onInstallStarted, wrong addon type"); + return false; + } + return undefined; + }, + + onInstallEnded: installEndedInstall => { + this._log.trace("_installAddon() - install ended for " + this.id); + gActiveInstallURLs.delete(installEndedInstall.sourceURI.spec); + + this._lastChangedDate = this._policy.now(); + this._startDate = this._policy.now(); + this._enabled = true; + + TelemetryLog.log(TELEMETRY_LOG.ACTIVATION_KEY, + [TELEMETRY_LOG.ACTIVATION.ACTIVATED, this.id]); + + let addon = installEndedInstall.addon; + this._name = addon.name; + this._addonId = addon.id; + this._description = addon.description || ""; + this._homepageURL = addon.homepageURL || ""; + + // Experiment add-ons default to userDisabled=true. Enable if needed. + if (addon.userDisabled) { + this._log.trace("Add-on is disabled. Enabling."); + listener._expectedID = addon.id; + AddonManager.addAddonListener(listener); + addon.userDisabled = false; + } else { + this._log.trace("Add-on is enabled. start() completed."); + deferred.resolve(); + } + }, + + onEnabled: addon => { + this._log.info("onEnabled() for " + addon.id); + + if (addon.id != listener._expectedID) { + return; + } + + AddonManager.removeAddonListener(listener); + deferred.resolve(); + }, + }; + + ["onDownloadCancelled", "onDownloadFailed", "onInstallCancelled", "onInstallFailed"] + .forEach(what => { + listener[what] = eventInstall => failureHandler(eventInstall, what) + }); + + install.addListener(listener); + install.install(); + + return yield deferred.promise; + }), + + /** + * Stop running the experiment if it is active. + * + * @param terminationKind (optional) + * The termination kind, e.g. ADDON_UNINSTALLED or EXPIRED. + * @param terminationReason (optional) + * The termination reason details for termination kind RECHECK. + * @return Promise<> Resolved when the operation is complete. + */ + stop: Task.async(function* (terminationKind, terminationReason) { + this._log.trace("stop() - id=" + this.id + ", terminationKind=" + terminationKind); + if (!this._enabled) { + throw new Error("Must not call stop() on an inactive experiment."); + } + + this._enabled = false; + let now = this._policy.now(); + this._lastChangedDate = now; + this._endDate = now; + + let changes = yield this.reconcileAddonState(); + this._logTermination(terminationKind, terminationReason); + + if (terminationKind == TELEMETRY_LOG.TERMINATION.ADDON_UNINSTALLED) { + changes |= this.ADDON_CHANGE_UNINSTALL; + } + + return changes; + }), + + /** + * Reconcile the state of the add-on against what it's supposed to be. + * + * If we are active, ensure the add-on is enabled and up to date. + * + * If we are inactive, ensure the add-on is not installed. + */ + reconcileAddonState: Task.async(function* () { + this._log.trace("reconcileAddonState()"); + + if (!this._enabled) { + if (!this._addonId) { + this._log.trace("reconcileAddonState() - Experiment is not enabled and " + + "has no add-on. Doing nothing."); + return this.ADDON_CHANGE_NONE; + } + + let addon = yield this._getAddon(); + if (!addon) { + this._log.trace("reconcileAddonState() - Inactive experiment has no " + + "add-on. Doing nothing."); + return this.ADDON_CHANGE_NONE; + } + + this._log.info("reconcileAddonState() - Uninstalling add-on for inactive " + + "experiment: " + addon.id); + gActiveUninstallAddonIDs.add(addon.id); + yield uninstallAddons([addon]); + gActiveUninstallAddonIDs.delete(addon.id); + return this.ADDON_CHANGE_UNINSTALL; + } + + // If we get here, we're supposed to be active. + + let changes = 0; + + // That requires an add-on. + let currentAddon = yield this._getAddon(); + + // If we have an add-on but it isn't up to date, uninstall it + // (to prepare for reinstall). + if (currentAddon && this._needsUpdate) { + this._log.info("reconcileAddonState() - Uninstalling add-on because update " + + "needed: " + currentAddon.id); + gActiveUninstallAddonIDs.add(currentAddon.id); + yield uninstallAddons([currentAddon]); + gActiveUninstallAddonIDs.delete(currentAddon.id); + changes |= this.ADDON_CHANGE_UNINSTALL; + } + + if (!currentAddon || this._needsUpdate) { + this._log.info("reconcileAddonState() - Installing add-on."); + yield this._installAddon(); + changes |= this.ADDON_CHANGE_INSTALL; + } + + let addon = yield this._getAddon(); + if (!addon) { + throw new Error("Could not obtain add-on for experiment that should be " + + "enabled."); + } + + // If we have the add-on and it is enabled, we are done. + if (!addon.userDisabled) { + return changes; + } + + // Check permissions to see if we can enable the addon. + if (!(addon.permissions & AddonManager.PERM_CAN_ENABLE)) { + throw new Error("Don't have permission to enable addon " + addon.id + ", perm=" + addon.permission); + } + + // Experiment addons should not require a restart. + if (addon.operationsRequiringRestart & AddonManager.OP_NEEDS_RESTART_ENABLE) { + throw new Error("Experiment addon requires a restart: " + addon.id); + } + + let deferred = Promise.defer(); + + // Else we need to enable it. + let listener = { + onEnabled: enabledAddon => { + if (enabledAddon.id != addon.id) { + return; + } + + AddonManager.removeAddonListener(listener); + deferred.resolve(); + }, + }; + + for (let handler of ["onDisabled", "onOperationCancelled", "onUninstalled"]) { + listener[handler] = (evtAddon) => { + if (evtAddon.id != addon.id) { + return; + } + + AddonManager.removeAddonListener(listener); + deferred.reject("Failed to enable addon " + addon.id + " due to: " + handler); + }; + } + + this._log.info("reconcileAddonState() - Activating add-on: " + addon.id); + AddonManager.addAddonListener(listener); + addon.userDisabled = false; + yield deferred.promise; + changes |= this.ADDON_CHANGE_ENABLE; + + this._log.info("reconcileAddonState() - Add-on has been enabled: " + addon.id); + return changes; + }), + + /** + * Obtain the underlying Addon from the Addon Manager. + * + * @return Promise<Addon|null> + */ + _getAddon() { + if (!this._addonId) { + return Promise.resolve(null); + } + + return AddonManager.getAddonByID(this._addonId).then(addon => { + if (addon && addon.appDisabled) { + // Don't return PreviousExperiments. + return null; + } + + return addon; + }); + }, + + _logTermination(terminationKind, terminationReason) { + if (terminationKind === undefined) { + return; + } + + if (!(terminationKind in TELEMETRY_LOG.TERMINATION)) { + this._log.warn("stop() - unknown terminationKind " + terminationKind); + return; + } + + let data = [terminationKind, this.id]; + if (terminationReason) { + data = data.concat(terminationReason); + } + + TelemetryLog.log(TELEMETRY_LOG.TERMINATION_KEY, data); + }, + + /** + * Determine whether an active experiment should be stopped. + */ + shouldStop() { + if (!this._enabled) { + throw new Error("shouldStop must not be called on disabled experiments."); + } + + let deferred = Promise.defer(); + this.isApplicable().then( + () => deferred.resolve({shouldStop: false}), + reason => deferred.resolve({shouldStop: true, reason}) + ); + + return deferred.promise; + }, + + /* + * Should this be discarded from the cache due to age? + */ + shouldDiscard() { + let limit = this._policy.now(); + limit.setDate(limit.getDate() - KEEP_HISTORY_N_DAYS); + return (this._lastChangedDate < limit); + }, + + /* + * Get next date (in epoch-ms) to schedule a re-evaluation for this. + * Returns 0 if it doesn't need one. + */ + getScheduleTime() { + if (this._enabled) { + let startTime = this._startDate.getTime(); + let maxActiveTime = startTime + 1000 * this._manifestData.maxActiveSeconds; + return Math.min(1000 * this._manifestData.endTime, maxActiveTime); + } + + if (this._endDate) { + return this._endDate.getTime(); + } + + return 1000 * this._manifestData.startTime; + }, + + /* + * Perform sanity checks on the experiment data. + */ + _isManifestDataValid(data) { + this._log.trace("isManifestDataValid() - data: " + JSON.stringify(data)); + + for (let key of this.MANIFEST_REQUIRED_FIELDS) { + if (!(key in data)) { + this._log.error("isManifestDataValid() - missing required key: " + key); + return false; + } + } + + for (let key in data) { + if (!this.MANIFEST_OPTIONAL_FIELDS.has(key) && + !this.MANIFEST_REQUIRED_FIELDS.has(key)) { + this._log.error("isManifestDataValid() - unknown key: " + key); + return false; + } + } + + return true; + }, +}; + +/** + * Strip a Date down to its UTC midnight. + * + * This will return a cloned Date object. The original is unchanged. + */ +var stripDateToMidnight = function(d) { + let m = new Date(d); + m.setUTCHours(0, 0, 0, 0); + + return m; +}; + +/** + * An Add-ons Manager provider that knows about old experiments. + * + * This provider exposes read-only add-ons corresponding to previously-active + * experiments. The existence of this provider (and the add-ons it knows about) + * facilitates the display of old experiments in the Add-ons Manager UI with + * very little custom code in that component. + */ +this.Experiments.PreviousExperimentProvider = function(experiments) { + this._experiments = experiments; + this._experimentList = []; + this._log = Log.repository.getLoggerWithMessagePrefix( + "Browser.Experiments.Experiments", + "PreviousExperimentProvider #" + gPreviousProviderCounter++ + "::"); +} + +this.Experiments.PreviousExperimentProvider.prototype = Object.freeze({ + name: "PreviousExperimentProvider", + + startup() { + this._log.trace("startup()"); + Services.obs.addObserver(this, EXPERIMENTS_CHANGED_TOPIC, false); + }, + + shutdown() { + this._log.trace("shutdown()"); + try { + Services.obs.removeObserver(this, EXPERIMENTS_CHANGED_TOPIC); + } catch (e) { + // Prevent crash in mochitest-browser3 on Mulet + } + }, + + observe(subject, topic, data) { + switch (topic) { + case EXPERIMENTS_CHANGED_TOPIC: + this._updateExperimentList(); + break; + } + }, + + getAddonByID(id, cb) { + for (let experiment of this._experimentList) { + if (experiment.id == id) { + cb(new PreviousExperimentAddon(experiment)); + return; + } + } + + cb(null); + }, + + getAddonsByTypes(types, cb) { + if (types && types.length > 0 && types.indexOf("experiment") == -1) { + cb([]); + return; + } + + cb(this._experimentList.map(e => new PreviousExperimentAddon(e))); + }, + + _updateExperimentList() { + return this._experiments.getExperiments().then((experiments) => { + let list = experiments.filter(e => !e.active); + + let newMap = new Map(list.map(e => [e.id, e])); + let oldMap = new Map(this._experimentList.map(e => [e.id, e])); + + let added = [...newMap.keys()].filter(id => !oldMap.has(id)); + let removed = [...oldMap.keys()].filter(id => !newMap.has(id)); + + for (let id of added) { + this._log.trace("updateExperimentList() - adding " + id); + let wrapper = new PreviousExperimentAddon(newMap.get(id)); + AddonManagerPrivate.callInstallListeners("onExternalInstall", null, wrapper, null, false); + AddonManagerPrivate.callAddonListeners("onInstalling", wrapper, false); + } + + for (let id of removed) { + this._log.trace("updateExperimentList() - removing " + id); + let wrapper = new PreviousExperimentAddon(oldMap.get(id)); + AddonManagerPrivate.callAddonListeners("onUninstalling", wrapper, false); + } + + this._experimentList = list; + + for (let id of added) { + let wrapper = new PreviousExperimentAddon(newMap.get(id)); + AddonManagerPrivate.callAddonListeners("onInstalled", wrapper); + } + + for (let id of removed) { + let wrapper = new PreviousExperimentAddon(oldMap.get(id)); + AddonManagerPrivate.callAddonListeners("onUninstalled", wrapper); + } + + return this._experimentList; + }); + }, +}); + +/** + * An add-on that represents a previously-installed experiment. + */ +function PreviousExperimentAddon(experiment) { + this._id = experiment.id; + this._name = experiment.name; + this._endDate = experiment.endDate; + this._description = experiment.description; +} + +PreviousExperimentAddon.prototype = Object.freeze({ + // BEGIN REQUIRED ADDON PROPERTIES + + get appDisabled() { + return true; + }, + + get blocklistState() { + Ci.nsIBlocklistService.STATE_NOT_BLOCKED + }, + + get creator() { + return new AddonManagerPrivate.AddonAuthor(""); + }, + + get foreignInstall() { + return false; + }, + + get id() { + return this._id; + }, + + get isActive() { + return false; + }, + + get isCompatible() { + return true; + }, + + get isPlatformCompatible() { + return true; + }, + + get name() { + return this._name; + }, + + get pendingOperations() { + return AddonManager.PENDING_NONE; + }, + + get permissions() { + return 0; + }, + + get providesUpdatesSecurely() { + return true; + }, + + get scope() { + return AddonManager.SCOPE_PROFILE; + }, + + get type() { + return "experiment"; + }, + + get userDisabled() { + return true; + }, + + get version() { + return null; + }, + + // END REQUIRED PROPERTIES + + // BEGIN OPTIONAL PROPERTIES + + get description() { + return this._description; + }, + + get updateDate() { + return new Date(this._endDate); + }, + + // END OPTIONAL PROPERTIES + + // BEGIN REQUIRED METHODS + + isCompatibleWith(appVersion, platformVersion) { + return true; + }, + + findUpdates(listener, reason, appVersion, platformVersion) { + AddonManagerPrivate.callNoUpdateListeners(this, listener, reason, + appVersion, platformVersion); + }, + + // END REQUIRED METHODS + + /** + * The end-date of the experiment, required for the Addon Manager UI. + */ + + get endDate() { + return this._endDate; + }, + +}); diff --git a/application/basilisk/experiments/Experiments.manifest b/application/basilisk/experiments/Experiments.manifest new file mode 100644 index 000000000..4a6a05a60 --- /dev/null +++ b/application/basilisk/experiments/Experiments.manifest @@ -0,0 +1,6 @@ +component {f7800463-3b97-47f9-9341-b7617e6d8d49} ExperimentsService.js +contract @mozilla.org/browser/experiments-service;1 {f7800463-3b97-47f9-9341-b7617e6d8d49} +category update-timer ExperimentsService @mozilla.org/browser/experiments-service;1,getService,experiments-update-timer,experiments.manifest.fetchIntervalSeconds,86400 +category profile-after-change ExperimentsService @mozilla.org/browser/experiments-service;1 + + diff --git a/application/basilisk/experiments/ExperimentsService.js b/application/basilisk/experiments/ExperimentsService.js new file mode 100644 index 000000000..28da659bb --- /dev/null +++ b/application/basilisk/experiments/ExperimentsService.js @@ -0,0 +1,118 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +const {interfaces: Ci, utils: Cu} = Components; + +Cu.import("resource://gre/modules/XPCOMUtils.jsm"); +Cu.import("resource://gre/modules/Services.jsm"); +Cu.import("resource://gre/modules/Preferences.jsm"); + +XPCOMUtils.defineLazyModuleGetter(this, "Experiments", + "resource:///modules/experiments/Experiments.jsm"); +XPCOMUtils.defineLazyModuleGetter(this, "OS", + "resource://gre/modules/osfile.jsm"); +XPCOMUtils.defineLazyModuleGetter(this, "CommonUtils", + "resource://services-common/utils.js"); + +const PREF_EXPERIMENTS_ENABLED = "experiments.enabled"; +const PREF_ACTIVE_EXPERIMENT = "experiments.activeExperiment"; // whether we have an active experiment +const PREF_TELEMETRY_ENABLED = "toolkit.telemetry.enabled"; +const PREF_TELEMETRY_UNIFIED = "toolkit.telemetry.unified"; +const DELAY_INIT_MS = 30 * 1000; + +// Whether the FHR/Telemetry unification features are enabled. +// Changing this pref requires a restart. +const IS_UNIFIED_TELEMETRY = Preferences.get(PREF_TELEMETRY_UNIFIED, false); + +XPCOMUtils.defineLazyGetter( + this, "gPrefs", () => { + return new Preferences(); + }); + +XPCOMUtils.defineLazyGetter( + this, "gExperimentsEnabled", () => { + // We can enable experiments if either unified Telemetry or FHR is on, and the user + // has opted into Telemetry. + return gPrefs.get(PREF_EXPERIMENTS_ENABLED, false) && + IS_UNIFIED_TELEMETRY && gPrefs.get(PREF_TELEMETRY_ENABLED, false); + }); + +XPCOMUtils.defineLazyGetter( + this, "gActiveExperiment", () => { + return gPrefs.get(PREF_ACTIVE_EXPERIMENT); + }); + +function ExperimentsService() { + this._initialized = false; + this._delayedInitTimer = null; +} + +ExperimentsService.prototype = { + classID: Components.ID("{f7800463-3b97-47f9-9341-b7617e6d8d49}"), + QueryInterface: XPCOMUtils.generateQI([Ci.nsITimerCallback, Ci.nsIObserver]), + + notify(timer) { + if (!gExperimentsEnabled) { + return; + } + if (OS.Constants.Path.profileDir === undefined) { + throw Error("Update timer fired before profile was initialized?"); + } + let instance = Experiments.instance(); + if (instance.isReady) { + instance.updateManifest(); + } + }, + + _delayedInit() { + if (!this._initialized) { + this._initialized = true; + Experiments.instance(); // for side effects + } + }, + + observe(subject, topic, data) { + switch (topic) { + case "profile-after-change": + if (gExperimentsEnabled) { + Services.obs.addObserver(this, "quit-application", false); + Services.obs.addObserver(this, "sessionstore-state-finalized", false); + Services.obs.addObserver(this, "EM-loaded", false); + + if (gActiveExperiment) { + this._initialized = true; + Experiments.instance(); // for side effects + } + } + break; + case "sessionstore-state-finalized": + if (!this._initialized) { + CommonUtils.namedTimer(this._delayedInit, DELAY_INIT_MS, this, "_delayedInitTimer"); + } + break; + case "EM-loaded": + if (!this._initialized) { + Experiments.instance(); // for side effects + this._initialized = true; + + if (this._delayedInitTimer) { + this._delayedInitTimer.clear(); + } + } + break; + case "quit-application": + Services.obs.removeObserver(this, "quit-application"); + Services.obs.removeObserver(this, "sessionstore-state-finalized"); + Services.obs.removeObserver(this, "EM-loaded"); + if (this._delayedInitTimer) { + this._delayedInitTimer.clear(); + } + break; + } + }, +}; + +this.NSGetFactory = XPCOMUtils.generateNSGetFactory([ExperimentsService]); diff --git a/application/basilisk/experiments/Makefile.in b/application/basilisk/experiments/Makefile.in new file mode 100644 index 000000000..5558582a6 --- /dev/null +++ b/application/basilisk/experiments/Makefile.in @@ -0,0 +1,16 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +include $(topsrcdir)/config/rules.mk + +# This is so hacky. Waiting on bug 988938. +addondir = $(srcdir)/test/addons +testdir = $(topobjdir)/_tests/xpcshell/browser/experiments/test/xpcshell + +misc:: $(call mkdir_deps,$(testdir)) + $(EXIT_ON_ERROR) \ + for dir in $(addondir)/*; do \ + base=`basename $$dir`; \ + (cd $$dir && zip -qr $(testdir)/$$base.xpi *); \ + done diff --git a/application/basilisk/experiments/docs/index.rst b/application/basilisk/experiments/docs/index.rst new file mode 100644 index 000000000..11e5d4faa --- /dev/null +++ b/application/basilisk/experiments/docs/index.rst @@ -0,0 +1,13 @@ +===================== +Telemetry Experiments +===================== + +Telemetry Experiments is a feature of Firefox that allows the installation +of add-ons called experiments to a subset of the Firefox population for +the purposes of experimenting with changes and collecting data on specific +aspects of application usage. + +.. toctree:: + :maxdepth: 1 + + manifest diff --git a/application/basilisk/experiments/docs/manifest.rst b/application/basilisk/experiments/docs/manifest.rst new file mode 100644 index 000000000..d4fad5243 --- /dev/null +++ b/application/basilisk/experiments/docs/manifest.rst @@ -0,0 +1,429 @@ +.. _experiments_manifests: + +===================== +Experiments Manifests +===================== + +*Experiments Manifests* are documents that describe the set of active +experiments a client may run. + +*Experiments Manifests* are fetched periodically by clients. When +fetched, clients look at the experiments within the manifest and +determine which experiments are applicable. If an experiment is +applicable, the client may download and start the experiment. + +Manifest Format +=============== + +Manifests are JSON documents where the main element is an object. + +The *schema* of the object is versioned and defined by the presence +of a top-level ``version`` property, whose integer value is the +schema version used by that manifest. Each version is documented +in the sections below. + +Version 1 +--------- + +Version 1 is the original manifest format. + +The following properties may exist in the root object: + +experiments + An array of objects describing candidate experiments. The format of + these objects is documented below. + + An array is used to create an explicit priority of experiments. + Experiments listed at the beginning of the array take priority over + experiments that follow. + +Experiments Objects +^^^^^^^^^^^^^^^^^^^ + +Each object in the ``experiments`` array may contain the following +properties: + +id + (required) String identifier of this experiment. The identifier should + be treated as opaque by clients. It is used to uniquely identify an + experiment for all of time. + +xpiURL + (required) String URL of the XPI that implements this experiment. + + If the experiment is activated, the client will download and install this + XPI. + +xpiHash + (required) String hash of the XPI that implements this experiment. + + The value is composed of a hash identifier followed by a colon + followed by the hash value. e.g. + `sha1:f677428b9172e22e9911039aef03f3736e7f78a7`. `sha1` and `sha256` + are the two supported hashing mechanisms. The hash value is the hex + encoding of the binary hash. + + When the client downloads the XPI for the experiment, it should compare + the hash of that XPI against this value. If the hashes don't match, + the client should not install the XPI. + + Clients may also use this hash as a means of determining when an + experiment's XPI has changed and should be refreshed. + +startTime + Integer seconds since UNIX epoch that this experiment should + start. Clients should not start an experiment if *now()* is less than + this value. + +maxStartTime + (optional) Integer seconds since UNIX epoch after which this experiment + should no longer start. + + Some experiments may wish to impose hard deadlines after which no new + clients should activate the experiment. This property may be used to + facilitate that. + +endTime + Integer seconds since UNIX epoch after which this experiment + should no longer run. Clients should cease an experiment when the current + time is beyond this value. + +maxActiveSeconds + Integer seconds defining the max wall time this experiment should be + active for. + + The client should deactivate the experiment this many seconds after + initial activation. + + This value only involves wall time, not browser activity or session time. + +appName + Array of application names this experiment should run on. + + An application name comes from ``nsIXULAppInfo.name``. It is a value + like ``Firefox``, ``Fennec``, or `B2G`. + + The client should compare its application name against the members of + this array. If a match is found, the experiment is applicable. + +minVersion + (optional) String version number of the minimum application version this + experiment should run on. + + A version number is something like ``27.0.0`` or ``28``. + + The client should compare its version number to this value. If the client's + version is greater or equal to this version (using a version-aware comparison + function), the experiment is applicable. + + If this is not specified, there is no lower bound to versions this + experiment should run on. + +maxVersion + (optional) String version number of the maximum application version this + experiment should run on. + + This is similar to ``minVersion`` except it sets the upper bound for + application versions. + + If the client's version is less than or equal to this version, the + experiment is applicable. + + If this is not specified, there is no upper bound to versions this + experiment should run on. + +version + (optional) Array of application versions this experiment should run on. + + This is similar to ``minVersion`` and ``maxVersion`` except only a + whitelisted set of specific versions are allowed. + + The client should compare its version to members of this array. If a match + is found, the experiment is applicable. + +minBuildID + (optional) String minimum Build ID this experiment should run on. + + Build IDs are values like ``201402261424``. + + The client should perform a string comparison of its Build ID against this + value. If its value is greater than or equal to this value, the experiment + is applicable. + +maxBuildID + (optional) String maximum Build ID this experiment should run on. + + This is similar to ``minBuildID`` except it sets the upper bound + for Build IDs. + + The client should perform a string comparison of its Build ID against + this value. If its value is less than or equal to this value, the + experiment is applicable. + +buildIDs + (optional) Array of Build IDs this experiment should run on. + + This is similar to ``minBuildID`` and ``maxBuildID`` except only a + whitelisted set of Build IDs are considered. + + The client should compare its Build ID to members of this array. If a + match is found, the experiment is applicable. + +os + (optional) Array of operating system identifiers this experiment should + run on. + + Values for this array come from ``nsIXULRuntime.OS``. + + The client will compare its operating system identifier to members + of this array. If a match is found, the experiment is applicable to the + client. + +channel + (optional) Array of release channel identifiers this experiment should run + on. + + The client will compare its channel to members of this array. If a match + is found, the experiment is applicable. + + If this property is not defined, the client should assume the experiment + is to run on all channels. + +locale + (optional) Array of locale identifiers this experiment should run on. + + A locale identifier is a string like ``en-US`` or ``zh-CN`` and is + obtained by looking at + ``nsIXULChromeRegistry.getSelectedLocale("global")``. + + The client should compare its locale identifier to members of this array. + If a match is found, the experiment is applicable. + + If this property is not defined, the client should assume the experiment + is to run on all locales. + +sample + (optional) Decimal number indicating the sampling rate for this experiment. + + This will contain a value between ``0.0`` and ``1.0``. The client should + generate a random decimal between ``0.0`` and ``1.0``. If the randomly + generated number is less than or equal to the value of this field, the + experiment is applicable. + +disabled + (optional) Boolean value indicating whether an experiment is disabled. + + Normally, experiments are deactivated after a certain time has passed or + after the experiment itself determines it no longer needs to run (perhaps + it collected sufficient data already). + + This property serves as a backup mechanism to remotely disable an + experiment before it was scheduled to be disabled. It can be used to + kill experiments that are found to be doing wrong or bad things or that + aren't useful. + + If this property is not defined or is false, the client should assume + the experiment is active and a candidate for activation. + +frozen + (optional) Boolean value indicating this experiment is frozen and no + longer accepting new enrollments. + + If a client sees a true value in this field, it should not attempt to + activate an experiment. + +jsfilter + (optional) JavaScript code that will be evaluated to determine experiment + applicability. + + This property contains the string representation of JavaScript code that + will be evaluated in a sandboxed environment using JavaScript's + ``eval()``. + + The string is expected to contain the definition of a JavaScript function + ``filter(context)``. This function receives as its argument an object + holding application state. See the section below for the definition of + this object. + + The purpose of this property is to allow experiments to define complex + rules and logic for evaluating experiment applicability in a manner + that is privacy conscious and doesn't require the transmission of + excessive data. + + The return value of this filter indicates whether the experiment is + applicable. Functions should return true if the experiment is + applicable. + + If an experiment is not applicable, they should throw an Error whose + message contains the reason the experiment is not applicable. This + message may be logged and sent to remote servers, so it should not + contain private or otherwise sensitive data that wouldn't normally + be submitted. + + If a falsey (or undefined) value is returned, the client should + assume the experiment is not applicable. + + If this property is not defined, the client does not consider a custom + JavaScript filter function when determining whether an experiment is + applicable. + +JavaScript Filter Context Objects +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The object passed to a ``jsfilter`` ``filter()`` function contains the +following properties: + +healthReportSubmissionEnabled + This property contains a boolean indicating whether Firefox Health + Report has its data submission flag enabled (whether Firefox Health + Report is sending data to remote servers). + +healthReportPayload + This property contains the current Firefox Health Report payload. + + The payload format is documented at :ref:`healthreport_dataformat`. + +telemetryPayload + This property contains the current Telemetry payload. + +The evaluation sandbox for the JavaScript filters may be destroyed +immediately after ``filter()`` returns. This function should not assume +async code will finish. + +Experiment Applicability and Client Behavior +============================================ + +The point of an experiment manifest is to define which experiments are +available and where and how to run them. This section explains those +rules in more detail. + +Many of the properties in *Experiment Objects* are related to determining +whether an experiment should run on a given client. This evaluation is +performed client side. + +1. Multiple conditions in an experiment +--------------------------------------- + +If multiple conditions are defined for an experiment, the client should +combine each condition with a logical *AND*: all conditions must be +satisfied for an experiment to run. If one condition fails, the experiment +is not applicable. + +2. Active experiment disappears from manifest +--------------------------------------------- + +If a specific experiment disappears from the manifest, the client should +continue conducting an already-active experiment. Furthermore, the +client should remember what the expiration events were for an experiment +and honor them. + +The rationale here is that we want to prevent an accidental deletion +or temporary failure on the server to inadvertantly deactivate +supposed-to-be-active experiments. We also don't want premature deletion +of an experiment from the manifest to result in indefinite activation +periods. + +3. Inactive experiment disappears from manifest +----------------------------------------------- + +If an inactive but scheduled-to-be-active experiment disappears from the +manifest, the client should not activate the experiment. + +If that experiment reappears in the manifest, the client should not +treat that experiment any differently than any other new experiment. Put +another way, the fact an inactive experiment disappears and then +reappears should not be significant. + +The rationale here is that server operators should have complete +control of an inactive experiment up to it's go-live date. + +4. Re-evaluating applicability on manifest refresh +-------------------------------------------------- + +When an experiment manifest is refreshed or updated, the client should +re-evaluate the applicability of each experiment therein. + +The rationale here is that the server may change the parameters of an +experiment and want clients to pick those up. + +5. Activating a previously non-applicable experiment +---------------------------------------------------- + +If the conditions of an experiment change or the state of the client +changes to allow an experiment to transition from previously +non-applicable to applicable, the experiment should be activated. + +For example, if a client is running version 28 and the experiment +initially requires version 29 or above, the client will not mark the +experiment as applicable. But if the client upgrades to version 29 or if +the manifest is updated to require 28 or above, the experiment will +become applicable. + +6. Deactivating a previously active experiment +---------------------------------------------- + +If the conditions of an experiment change or the state of the client +changes and an active experiment is no longer applicable, that +experiment should be deactivated. + +7. Calculation of sampling-based applicability +---------------------------------------------- + +For calculating sampling-based applicability, the client will associate +a random value between ``0.0`` and ``1.0`` for each observed experiment +ID. This random value will be generated the first time sampling +applicability is evaluated. This random value will be persisted and used +in future applicability evaluations for this experiment. + +By saving and re-using the value, the client is able to reliably and +consistently evaluate applicability, even if the sampling threshold +in the manifest changes. + +Clients should retain the randomly-generated sampling value for +experiments that no longer appear in a manifest for a period of at least +30 days. The rationale is that if an experiment disappears and reappears +from a manifest, the client will not have multiple opportunities to +generate a random value that satisfies the sampling criteria. + +8. Incompatible version numbers +------------------------------- + +If a client receives a manifest with a version number that it doesn't +recognize, it should ignore the manifest. + +9. Usage of old manifests +------------------------- + +If a client experiences an error fetching a manifest (server not +available) or if the manifest is corrupt, not readable, or compatible, +the client may use a previously-fetched (cached) manifest. + +10. Updating XPIs +----------------- + +If the URL or hash of an active experiment's XPI changes, the client +should fetch the new XPI, uninstall the old XPI, and install the new +XPI. + +Examples +======== + +Here is an example manifest:: + + { + "version": 1, + "experiments": [ + { + "id": "da9d7f4f-f3f9-4f81-bacd-6f0626ffa360", + "xpiURL": "https://experiments.mozilla.org/foo.xpi", + "xpiHash": "sha1:cb1eb32b89d86d78b7326f416cf404548c5e0099", + "startTime": 1393000000, + "endTime": 1394000000, + "appName": ["Firefox", "Fennec"], + "minVersion": "28", + "maxVersion": "30", + "os": ["windows", "linux", "osx"], + "jsfilter": "function filter(context) { return context.healthReportEnabled; }" + } + ] + } diff --git a/application/basilisk/experiments/moz.build b/application/basilisk/experiments/moz.build new file mode 100644 index 000000000..a11e4b725 --- /dev/null +++ b/application/basilisk/experiments/moz.build @@ -0,0 +1,18 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at http://mozilla.org/MPL/2.0/. + +HAS_MISC_RULE = True + +EXTRA_COMPONENTS += [ + 'Experiments.manifest', + 'ExperimentsService.js', +] + +EXTRA_JS_MODULES.experiments += [ + 'Experiments.jsm', +] + +XPCSHELL_TESTS_MANIFESTS += ['test/xpcshell/xpcshell.ini'] + +SPHINX_TREES['experiments'] = 'docs' diff --git a/application/basilisk/experiments/test/addons/experiment-1/install.rdf b/application/basilisk/experiments/test/addons/experiment-1/install.rdf new file mode 100644 index 000000000..f9d70054a --- /dev/null +++ b/application/basilisk/experiments/test/addons/experiment-1/install.rdf @@ -0,0 +1,16 @@ +<?xml version="1.0"?> + +<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns:em="http://www.mozilla.org/2004/em-rdf#"> + + <Description about="urn:mozilla:install-manifest"> + <em:id>test-experiment-1@tests.mozilla.org</em:id> + <em:version>1</em:version> + <em:type>128</em:type> + + <!-- Front End MetaData --> + <em:name>Test experiment 1</em:name> + <em:description>Yet another experiment that experiments experimentally.</em:description> + + </Description> +</RDF> diff --git a/application/basilisk/experiments/test/addons/experiment-1a/install.rdf b/application/basilisk/experiments/test/addons/experiment-1a/install.rdf new file mode 100644 index 000000000..7806b11b1 --- /dev/null +++ b/application/basilisk/experiments/test/addons/experiment-1a/install.rdf @@ -0,0 +1,16 @@ +<?xml version="1.0"?> + +<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns:em="http://www.mozilla.org/2004/em-rdf#"> + + <Description about="urn:mozilla:install-manifest"> + <em:id>test-experiment-1@tests.mozilla.org</em:id> + <em:version>1.1</em:version> + <em:type>128</em:type> + + <!-- Front End MetaData --> + <em:name>Test experiment 1.1</em:name> + <em:description>And yet another experiment that experiments experimentally.</em:description> + + </Description> +</RDF> diff --git a/application/basilisk/experiments/test/addons/experiment-2/install.rdf b/application/basilisk/experiments/test/addons/experiment-2/install.rdf new file mode 100644 index 000000000..69122c0ef --- /dev/null +++ b/application/basilisk/experiments/test/addons/experiment-2/install.rdf @@ -0,0 +1,16 @@ +<?xml version="1.0"?> + +<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#" + xmlns:em="http://www.mozilla.org/2004/em-rdf#"> + + <Description about="urn:mozilla:install-manifest"> + <em:id>test-experiment-2@tests.mozilla.org</em:id> + <em:version>1</em:version> + <em:type>128</em:type> + + <!-- Front End MetaData --> + <em:name>Test experiment 2</em:name> + <em:description>And yet another experiment that experiments experimentally.</em:description> + + </Description> +</RDF> diff --git a/application/basilisk/experiments/test/addons/experiment-racybranch/bootstrap.js b/application/basilisk/experiments/test/addons/experiment-racybranch/bootstrap.js new file mode 100644 index 000000000..d77fd4345 --- /dev/null +++ b/application/basilisk/experiments/test/addons/experiment-racybranch/bootstrap.js @@ -0,0 +1,35 @@ +/* exported startup, shutdown, install, uninstall */ + +var {classes: Cc, interfaces: Ci, utils: Cu} = Components; + +Cu.import("resource:///modules/experiments/Experiments.jsm"); + +var gStarted = false; + +function startup(data, reasonCode) { + if (gStarted) { + return; + } + gStarted = true; + + // delay realstartup to trigger the race condition + Cc["@mozilla.org/thread-manager;1"].getService(Ci.nsIThreadManager) + .mainThread.dispatch(realstartup, 0); +} + +function realstartup() { + let experiments = Experiments.instance(); + let experiment = experiments._getActiveExperiment(); + if (experiment.branch) { + Cu.reportError("Found pre-existing branch: " + experiment.branch); + return; + } + + let branch = "racy-set"; + experiments.setExperimentBranch(experiment.id, branch) + .then(null, Cu.reportError); +} + +function shutdown() { } +function install() { } +function uninstall() { } diff --git a/application/basilisk/experiments/test/addons/experiment-racybranch/install.rdf b/application/basilisk/experiments/test/addons/experiment-racybranch/install.rdf new file mode 100644 index 000000000..cebaede56 --- /dev/null +++ b/application/basilisk/experiments/test/addons/experiment-racybranch/install.rdf @@ -0,0 +1,16 @@ +<?xml version="1.0"?>
+
+<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+ xmlns:em="http://www.mozilla.org/2004/em-rdf#">
+
+ <Description about="urn:mozilla:install-manifest">
+ <em:id>test-experiment-racybranch@tests.mozilla.org</em:id>
+ <em:version>1</em:version>
+ <em:type>128</em:type>
+
+ <!-- Front End MetaData -->
+ <em:name>Test experiment racybranch</em:name>
+ <em:description>An experiment that sets the experiment branch in a potentially racy way.</em:description>
+
+ </Description>
+</RDF>
diff --git a/application/basilisk/experiments/test/xpcshell/.eslintrc.js b/application/basilisk/experiments/test/xpcshell/.eslintrc.js new file mode 100644 index 000000000..1f540a05b --- /dev/null +++ b/application/basilisk/experiments/test/xpcshell/.eslintrc.js @@ -0,0 +1,15 @@ +"use strict"; + +module.exports = { + "extends": [ + "../../../../testing/xpcshell/xpcshell.eslintrc.js" + ], + + "rules": { + "no-unused-vars": ["error", { + "vars": "all", + "varsIgnorePattern": "^(Cc|Ci|Cr|Cu|EXPORTED_SYMBOLS)$", + "args": "none" + }] + } +}; diff --git a/application/basilisk/experiments/test/xpcshell/experiments_1.manifest b/application/basilisk/experiments/test/xpcshell/experiments_1.manifest new file mode 100644 index 000000000..0401ea328 --- /dev/null +++ b/application/basilisk/experiments/test/xpcshell/experiments_1.manifest @@ -0,0 +1,19 @@ +{ + "version": 1, + "experiments": [ + { + "id": "test-experiment-1@tests.mozilla.org", + "xpiURL": "https://experiments.mozilla.org/foo.xpi", + "xpiHash": "sha1:cb1eb32b89d86d78b7326f416cf404548c5e0099", + "startTime": 1393000000, + "endTime": 1394000000, + "appName": ["Firefox", "Fennec"], + "minVersion": "28", + "maxVersion": "30", + "maxActiveSeconds": 60, + "os": ["windows", "linux", "osx"], + "channel": ["daily", "weekly", "nightly"], + "jsfilter": "function filter(context) { return true; }" + } + ] +} diff --git a/application/basilisk/experiments/test/xpcshell/head.js b/application/basilisk/experiments/test/xpcshell/head.js new file mode 100644 index 000000000..14067d60b --- /dev/null +++ b/application/basilisk/experiments/test/xpcshell/head.js @@ -0,0 +1,199 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* exported PREF_EXPERIMENTS_ENABLED, PREF_LOGGING_LEVEL, PREF_LOGGING_DUMP + PREF_MANIFEST_URI, PREF_FETCHINTERVAL, EXPERIMENT1_ID, + EXPERIMENT1_NAME, EXPERIMENT1_XPI_SHA1, EXPERIMENT1A_NAME, + EXPERIMENT1A_XPI_SHA1, EXPERIMENT2_ID, EXPERIMENT2_XPI_SHA1, + EXPERIMENT3_ID, EXPERIMENT4_ID, FAKE_EXPERIMENTS_1, + FAKE_EXPERIMENTS_2, gAppInfo, removeCacheFile, defineNow, + futureDate, dateToSeconds, loadAddonManager, promiseRestartManager, + startAddonManagerOnly, getExperimentAddons, replaceExperiments */ + +var {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components; + +Cu.import("resource://gre/modules/Services.jsm"); +Cu.import("resource://gre/modules/XPCOMUtils.jsm"); +Cu.import("resource://gre/modules/Promise.jsm"); +Cu.import("resource://gre/modules/Task.jsm"); +Cu.import("resource://gre/modules/osfile.jsm"); +Cu.import("resource://testing-common/AddonManagerTesting.jsm"); +Cu.import("resource://testing-common/AddonTestUtils.jsm"); + +XPCOMUtils.defineLazyModuleGetter(this, "AddonManager", + "resource://gre/modules/AddonManager.jsm"); + +const PREF_EXPERIMENTS_ENABLED = "experiments.enabled"; +const PREF_LOGGING_LEVEL = "experiments.logging.level"; +const PREF_LOGGING_DUMP = "experiments.logging.dump"; +const PREF_MANIFEST_URI = "experiments.manifest.uri"; +const PREF_FETCHINTERVAL = "experiments.manifest.fetchIntervalSeconds"; +const PREF_TELEMETRY_ENABLED = "toolkit.telemetry.enabled"; + +function getExperimentPath(base) { + let p = do_get_cwd(); + p.append(base); + return p.path; +} + +function sha1File(path) { + let f = Cc["@mozilla.org/file/local;1"] + .createInstance(Ci.nsILocalFile); + f.initWithPath(path); + let hasher = Cc["@mozilla.org/security/hash;1"] + .createInstance(Ci.nsICryptoHash); + hasher.init(hasher.SHA1); + + let is = Cc["@mozilla.org/network/file-input-stream;1"] + .createInstance(Ci.nsIFileInputStream); + is.init(f, -1, 0, 0); + hasher.updateFromStream(is, Math.pow(2, 32) - 1); + is.close(); + let bytes = hasher.finish(false); + + let rv = ""; + for (let i = 0; i < bytes.length; i++) { + rv += ("0" + bytes.charCodeAt(i).toString(16)).substr(-2); + } + return rv; +} + +const EXPERIMENT1_ID = "test-experiment-1@tests.mozilla.org"; +const EXPERIMENT1_XPI_NAME = "experiment-1.xpi"; +const EXPERIMENT1_NAME = "Test experiment 1"; +const EXPERIMENT1_PATH = getExperimentPath(EXPERIMENT1_XPI_NAME); +const EXPERIMENT1_XPI_SHA1 = "sha1:" + sha1File(EXPERIMENT1_PATH); + + +const EXPERIMENT1A_XPI_NAME = "experiment-1a.xpi"; +const EXPERIMENT1A_NAME = "Test experiment 1.1"; +const EXPERIMENT1A_PATH = getExperimentPath(EXPERIMENT1A_XPI_NAME); +const EXPERIMENT1A_XPI_SHA1 = "sha1:" + sha1File(EXPERIMENT1A_PATH); + +const EXPERIMENT2_ID = "test-experiment-2@tests.mozilla.org" +const EXPERIMENT2_XPI_NAME = "experiment-2.xpi"; +const EXPERIMENT2_PATH = getExperimentPath(EXPERIMENT2_XPI_NAME); +const EXPERIMENT2_XPI_SHA1 = "sha1:" + sha1File(EXPERIMENT2_PATH); + +const EXPERIMENT3_ID = "test-experiment-3@tests.mozilla.org"; +const EXPERIMENT4_ID = "test-experiment-4@tests.mozilla.org"; + +const FAKE_EXPERIMENTS_1 = [ + { + id: "id1", + name: "experiment1", + description: "experiment 1", + active: true, + detailUrl: "https://dummy/experiment1", + branch: "foo", + }, +]; + +const FAKE_EXPERIMENTS_2 = [ + { + id: "id2", + name: "experiment2", + description: "experiment 2", + active: false, + endDate: new Date(2014, 2, 11, 2, 4, 35, 42).getTime(), + detailUrl: "https://dummy/experiment2", + branch: null, + }, + { + id: "id1", + name: "experiment1", + description: "experiment 1", + active: false, + endDate: new Date(2014, 2, 10, 0, 0, 0, 0).getTime(), + detailURL: "https://dummy/experiment1", + branch: null, + }, +]; + +var gAppInfo = null; + +function removeCacheFile() { + let path = OS.Path.join(OS.Constants.Path.profileDir, "experiments.json"); + return OS.File.remove(path); +} + +function patchPolicy(policy, data) { + for (let key of Object.keys(data)) { + Object.defineProperty(policy, key, { + value: data[key], + writable: true, + }); + } +} + +function defineNow(policy, time) { + patchPolicy(policy, { now: () => new Date(time) }); +} + +function futureDate(date, offset) { + return new Date(date.getTime() + offset); +} + +function dateToSeconds(date) { + return date.getTime() / 1000; +} + +var gGlobalScope = this; +function loadAddonManager() { + AddonTestUtils.init(gGlobalScope); + AddonTestUtils.overrideCertDB(); + createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2"); + return AddonTestUtils.promiseStartupManager(); +} + +const { + promiseRestartManager, +} = AddonTestUtils; + +// Starts the addon manager without creating app info. We can't directly use +// |loadAddonManager| defined above in test_conditions.js as it would make the test fail. +function startAddonManagerOnly() { + let addonManager = Cc["@mozilla.org/addons/integration;1"] + .getService(Ci.nsIObserver) + .QueryInterface(Ci.nsITimerCallback); + addonManager.observe(null, "addons-startup", null); +} + +function getExperimentAddons(previous = false) { + let deferred = Promise.defer(); + + AddonManager.getAddonsByTypes(["experiment"], (addons) => { + if (previous) { + deferred.resolve(addons); + } else { + deferred.resolve(addons.filter(a => !a.appDisabled)); + } + }); + + return deferred.promise; +} + +function createAppInfo(ID = "xpcshell@tests.mozilla.org", name = "XPCShell", + version = "1.0", platformVersion = "1.0") { + AddonTestUtils.createAppInfo(ID, name, version, platformVersion); + gAppInfo = AddonTestUtils.appInfo; +} + +/** + * Replace the experiments on an Experiments with a new list. + * + * This monkeypatches getExperiments(). It doesn't monkeypatch the internal + * experiments list. So its utility is not as great as it could be. + */ +function replaceExperiments(experiment, list) { + Object.defineProperty(experiment, "getExperiments", { + writable: true, + value: () => { + return Promise.resolve(list); + }, + }); +} + +// Experiments require Telemetry to be enabled, and that's not true for debug +// builds. Let's just enable it here instead of going through each test. +Services.prefs.setBoolPref(PREF_TELEMETRY_ENABLED, true); diff --git a/application/basilisk/experiments/test/xpcshell/test_activate.js b/application/basilisk/experiments/test/xpcshell/test_activate.js new file mode 100644 index 000000000..fbca2ea34 --- /dev/null +++ b/application/basilisk/experiments/test/xpcshell/test_activate.js @@ -0,0 +1,151 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +Cu.import("resource://testing-common/httpd.js"); +Cu.import("resource:///modules/experiments/Experiments.jsm"); + +const SEC_IN_ONE_DAY = 24 * 60 * 60; +const MS_IN_ONE_DAY = SEC_IN_ONE_DAY * 1000; + +var gHttpServer = null; +var gHttpRoot = null; +var gPolicy = null; + +function ManifestEntry(data) { + this.id = data.id || EXPERIMENT1_ID; + this.xpiURL = data.xpiURL || gHttpRoot + EXPERIMENT1_XPI_NAME; + this.xpiHash = data.xpiHash || EXPERIMENT1_XPI_SHA1; + this.appName = data.appName || ["XPCShell"]; + this.channel = data.appName || ["nightly"]; + this.startTime = data.startTime || new Date(2010, 0, 1, 12).getTime() / 1000; + this.endTime = data.endTime || new Date(9001, 0, 1, 12).getTime() / 1000; + this.maxActiveSeconds = data.maxActiveSeconds || 5 * SEC_IN_ONE_DAY; +} + +function run_test() { + run_next_test(); +} + +add_task(function* test_setup() { + loadAddonManager(); + gPolicy = new Experiments.Policy(); + + gHttpServer = new HttpServer(); + gHttpServer.start(-1); + let port = gHttpServer.identity.primaryPort; + gHttpRoot = "http://localhost:" + port + "/"; + gHttpServer.registerDirectory("/", do_get_cwd()); + do_register_cleanup(() => gHttpServer.stop(() => {})); + + patchPolicy(gPolicy, { + updatechannel: () => "nightly", + }); + + Services.prefs.setBoolPref(PREF_EXPERIMENTS_ENABLED, true); + Services.prefs.setIntPref(PREF_LOGGING_LEVEL, 0); + Services.prefs.setBoolPref(PREF_LOGGING_DUMP, true); +}); + +function isApplicable(experiment) { + let deferred = Promise.defer(); + experiment.isApplicable().then( + result => deferred.resolve({ applicable: true, reason: null }), + reason => deferred.resolve({ applicable: false, reason }) + ); + + return deferred.promise; +} + +add_task(function* test_startStop() { + let baseDate = new Date(2014, 5, 1, 12); + let startDate = futureDate(baseDate, 30 * MS_IN_ONE_DAY); + let endDate = futureDate(baseDate, 60 * MS_IN_ONE_DAY); + let manifestData = new ManifestEntry({ + startTime: dateToSeconds(startDate), + endTime: dateToSeconds(endDate), + maxActiveSeconds: 10 * SEC_IN_ONE_DAY, + }); + let experiment = new Experiments.ExperimentEntry(gPolicy); + experiment.initFromManifestData(manifestData); + + // We need to associate it with the singleton so the onInstallStarted + // Addon Manager listener will know about it. + Experiments.instance()._experiments = new Map(); + Experiments.instance()._experiments.set(experiment.id, experiment); + + let result; + + defineNow(gPolicy, baseDate); + result = yield isApplicable(experiment); + Assert.equal(result.applicable, false, "Experiment should not be applicable."); + Assert.equal(experiment.enabled, false, "Experiment should not be enabled."); + + let addons = yield getExperimentAddons(); + Assert.equal(addons.length, 0, "No experiment add-ons are installed."); + + defineNow(gPolicy, futureDate(startDate, 5 * MS_IN_ONE_DAY)); + result = yield isApplicable(experiment); + Assert.equal(result.applicable, true, "Experiment should now be applicable."); + Assert.equal(experiment.enabled, false, "Experiment should not be enabled."); + + let changes = yield experiment.start(); + Assert.equal(changes, experiment.ADDON_CHANGE_INSTALL, "Add-on was installed."); + addons = yield getExperimentAddons(); + Assert.equal(experiment.enabled, true, "Experiment should now be enabled."); + Assert.equal(addons.length, 1, "1 experiment add-on is installed."); + Assert.equal(addons[0].id, experiment._addonId, "The add-on is the one we expect."); + Assert.equal(addons[0].userDisabled, false, "The add-on is not userDisabled."); + Assert.ok(addons[0].isActive, "The add-on is active."); + + changes = yield experiment.stop(); + Assert.equal(changes, experiment.ADDON_CHANGE_UNINSTALL, "Add-on was uninstalled."); + addons = yield getExperimentAddons(); + Assert.equal(experiment.enabled, false, "Experiment should not be enabled."); + Assert.equal(addons.length, 0, "Experiment should be uninstalled from the Addon Manager."); + + changes = yield experiment.start(); + Assert.equal(changes, experiment.ADDON_CHANGE_INSTALL, "Add-on was installed."); + addons = yield getExperimentAddons(); + Assert.equal(experiment.enabled, true, "Experiment should now be enabled."); + Assert.equal(addons.length, 1, "1 experiment add-on is installed."); + Assert.equal(addons[0].id, experiment._addonId, "The add-on is the one we expect."); + Assert.equal(addons[0].userDisabled, false, "The add-on is not userDisabled."); + Assert.ok(addons[0].isActive, "The add-on is active."); + + result = yield experiment.shouldStop(); + Assert.equal(result.shouldStop, false, "shouldStop should be false."); + Assert.equal(experiment.enabled, true, "Experiment should be enabled."); + addons = yield getExperimentAddons(); + Assert.equal(addons.length, 1, "Experiment still in add-ons manager."); + Assert.ok(addons[0].isActive, "The add-on is still active."); + + defineNow(gPolicy, futureDate(endDate, MS_IN_ONE_DAY)); + result = yield experiment.shouldStop(); + Assert.equal(result.shouldStop, true, "shouldStop should now be true."); + changes = yield experiment.stop(); + Assert.equal(changes, experiment.ADDON_CHANGE_UNINSTALL, "Add-on should be uninstalled."); + Assert.equal(experiment.enabled, false, "Experiment should be disabled."); + addons = yield getExperimentAddons(); + Assert.equal(addons.length, 0, "Experiment add-on is uninstalled."); + + // Ensure hash validation works. + // We set an incorrect hash and expect the install to fail. + experiment._manifestData.xpiHash = "sha1:41014dcc66b4dcedcd973491a1530a32f0517d8a"; + let errored = false; + try { + yield experiment.start(); + } catch (ex) { + errored = true; + } + Assert.ok(experiment._failedStart, "Experiment failed to start."); + Assert.ok(errored, "start() threw an exception."); + + // Make sure "ignore hashes" mode works. + gPolicy.ignoreHashes = true; + changes = yield experiment.start(); + Assert.equal(changes, experiment.ADDON_CHANGE_INSTALL); + yield experiment.stop(); + gPolicy.ignoreHashes = false; +}); diff --git a/application/basilisk/experiments/test/xpcshell/test_api.js b/application/basilisk/experiments/test/xpcshell/test_api.js new file mode 100644 index 000000000..d494f7baa --- /dev/null +++ b/application/basilisk/experiments/test/xpcshell/test_api.js @@ -0,0 +1,1646 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +Cu.import("resource://testing-common/httpd.js"); +Cu.import("resource://testing-common/AddonManagerTesting.jsm"); + +XPCOMUtils.defineLazyModuleGetter(this, "Experiments", + "resource:///modules/experiments/Experiments.jsm"); + +const MANIFEST_HANDLER = "manifests/handler"; + +const SEC_IN_ONE_DAY = 24 * 60 * 60; +const MS_IN_ONE_DAY = SEC_IN_ONE_DAY * 1000; + +var gHttpServer = null; +var gHttpRoot = null; +var gDataRoot = null; +var gPolicy = null; +var gManifestObject = null; +var gManifestHandlerURI = null; +var gTimerScheduleOffset = -1; + +function uninstallExperimentAddons() { + return Task.spawn(function* () { + let addons = yield getExperimentAddons(); + for (let a of addons) { + yield AddonManagerTesting.uninstallAddonByID(a.id); + } + }); +} + +function testCleanup(experimentsInstance) { + return Task.spawn(function* () { + yield promiseRestartManager(); + yield uninstallExperimentAddons(); + yield removeCacheFile(); + }); +} + +function run_test() { + run_next_test(); +} + +add_task(function* test_setup() { + loadAddonManager(); + + gHttpServer = new HttpServer(); + gHttpServer.start(-1); + let port = gHttpServer.identity.primaryPort; + gHttpRoot = "http://localhost:" + port + "/"; + gDataRoot = gHttpRoot + "data/"; + gManifestHandlerURI = gHttpRoot + MANIFEST_HANDLER; + gHttpServer.registerDirectory("/data/", do_get_cwd()); + gHttpServer.registerPathHandler("/" + MANIFEST_HANDLER, (request, response) => { + response.setStatusLine(null, 200, "OK"); + response.write(JSON.stringify(gManifestObject)); + response.processAsync(); + response.finish(); + }); + do_register_cleanup(() => gHttpServer.stop(() => {})); + + Services.prefs.setBoolPref(PREF_EXPERIMENTS_ENABLED, true); + Services.prefs.setIntPref(PREF_LOGGING_LEVEL, 0); + Services.prefs.setBoolPref(PREF_LOGGING_DUMP, true); + Services.prefs.setCharPref(PREF_MANIFEST_URI, gManifestHandlerURI); + Services.prefs.setIntPref(PREF_FETCHINTERVAL, 0); + + gPolicy = new Experiments.Policy(); + patchPolicy(gPolicy, { + updatechannel: () => "nightly", + oneshotTimer: (callback, timeout, thisObj, name) => gTimerScheduleOffset = timeout, + }); +}); + +add_task(function* test_contract() { + Cc["@mozilla.org/browser/experiments-service;1"].getService(); +}); + +// Test basic starting and stopping of experiments. + +add_task(function* test_getExperiments() { + const OBSERVER_TOPIC = "experiments-changed"; + let observerFireCount = 0; + let expectedObserverFireCount = 0; + let observer = () => ++observerFireCount; + Services.obs.addObserver(observer, OBSERVER_TOPIC, false); + + // Dates the following tests are based on. + + let baseDate = new Date(2014, 5, 1, 12); + let startDate1 = futureDate(baseDate, 50 * MS_IN_ONE_DAY); + let endDate1 = futureDate(baseDate, 100 * MS_IN_ONE_DAY); + let startDate2 = futureDate(baseDate, 150 * MS_IN_ONE_DAY); + let endDate2 = futureDate(baseDate, 200 * MS_IN_ONE_DAY); + + // The manifest data we test with. + + gManifestObject = { + "version": 1, + experiments: [ + { + id: EXPERIMENT2_ID, + xpiURL: gDataRoot + EXPERIMENT2_XPI_NAME, + xpiHash: EXPERIMENT2_XPI_SHA1, + startTime: dateToSeconds(startDate2), + endTime: dateToSeconds(endDate2), + maxActiveSeconds: 10 * SEC_IN_ONE_DAY, + appName: ["XPCShell"], + channel: ["nightly"], + }, + { + id: EXPERIMENT1_ID, + xpiURL: gDataRoot + EXPERIMENT1_XPI_NAME, + xpiHash: EXPERIMENT1_XPI_SHA1, + startTime: dateToSeconds(startDate1), + endTime: dateToSeconds(endDate1), + maxActiveSeconds: 10 * SEC_IN_ONE_DAY, + appName: ["XPCShell"], + channel: ["nightly"], + }, + ], + }; + + // Data to compare the result of Experiments.getExperiments() against. + + let experimentListData = [ + { + id: EXPERIMENT2_ID, + name: "Test experiment 2", + description: "And yet another experiment that experiments experimentally.", + }, + { + id: EXPERIMENT1_ID, + name: EXPERIMENT1_NAME, + description: "Yet another experiment that experiments experimentally.", + }, + ]; + + let experiments = new Experiments.Experiments(gPolicy); + + // Trigger update, clock set to before any activation. + // Use updateManifest() to provide for coverage of that path. + + let now = baseDate; + gTimerScheduleOffset = -1; + defineNow(gPolicy, now); + + yield experiments.updateManifest(); + Assert.equal(observerFireCount, ++expectedObserverFireCount, + "Experiments observer should have been called."); + Assert.equal(experiments.getActiveExperimentID(), null, + "getActiveExperimentID should return null"); + + let list = yield experiments.getExperiments(); + Assert.equal(list.length, 0, "Experiment list should be empty."); + let addons = yield getExperimentAddons(); + Assert.equal(addons.length, 0, "Precondition: No experiment add-ons are installed."); + + try { + yield experiments.getExperimentBranch(); + Assert.ok(false, "getExperimentBranch should fail with no experiment"); + } catch (e) { + Assert.ok(true, "getExperimentBranch correctly threw"); + } + + // Trigger update, clock set for experiment 1 to start. + + now = futureDate(startDate1, 5 * MS_IN_ONE_DAY); + gTimerScheduleOffset = -1; + defineNow(gPolicy, now); + + yield experiments.updateManifest(); + Assert.equal(observerFireCount, ++expectedObserverFireCount, + "Experiments observer should have been called."); + + Assert.equal(experiments.getActiveExperimentID(), EXPERIMENT1_ID, + "getActiveExperimentID should return the active experiment1"); + + list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "Experiment list should have 1 entry now."); + addons = yield getExperimentAddons(); + Assert.equal(addons.length, 1, "An experiment add-on was installed."); + + experimentListData[1].active = true; + experimentListData[1].endDate = now.getTime() + 10 * MS_IN_ONE_DAY; + for (let k of Object.keys(experimentListData[1])) { + Assert.equal(experimentListData[1][k], list[0][k], + "Property " + k + " should match reference data."); + } + + let b = yield experiments.getExperimentBranch(); + Assert.strictEqual(b, null, "getExperimentBranch should return null by default"); + + b = yield experiments.getExperimentBranch(EXPERIMENT1_ID); + Assert.strictEqual(b, null, "getExperimentsBranch should return null (with id)"); + + yield experiments.setExperimentBranch(EXPERIMENT1_ID, "foo"); + b = yield experiments.getExperimentBranch(); + Assert.strictEqual(b, "foo", "getExperimentsBranch should return the set value"); + + Assert.equal(observerFireCount, ++expectedObserverFireCount, + "Experiments observer should have been called."); + + Assert.equal(gTimerScheduleOffset, 10 * MS_IN_ONE_DAY, + "Experiment re-evaluation should have been scheduled correctly."); + + // Trigger update, clock set for experiment 1 to stop. + + now = futureDate(endDate1, 1000); + gTimerScheduleOffset = -1; + defineNow(gPolicy, now); + + yield experiments.updateManifest(); + Assert.equal(observerFireCount, ++expectedObserverFireCount, + "Experiments observer should have been called."); + + Assert.equal(experiments.getActiveExperimentID(), null, + "getActiveExperimentID should return null again"); + + list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "Experiment list should have 1 entry."); + addons = yield getExperimentAddons(); + Assert.equal(addons.length, 0, "The experiment add-on should be uninstalled."); + + experimentListData[1].active = false; + experimentListData[1].endDate = now.getTime(); + for (let k of Object.keys(experimentListData[1])) { + Assert.equal(experimentListData[1][k], list[0][k], + "Property " + k + " should match reference data."); + } + + Assert.equal(gTimerScheduleOffset, startDate2 - now, + "Experiment re-evaluation should have been scheduled correctly."); + + // Trigger update, clock set for experiment 2 to start. + // Use notify() to provide for coverage of that path. + + now = startDate2; + gTimerScheduleOffset = -1; + defineNow(gPolicy, now); + + yield experiments.notify(); + Assert.equal(observerFireCount, ++expectedObserverFireCount, + "Experiments observer should have been called."); + + Assert.equal(experiments.getActiveExperimentID(), EXPERIMENT2_ID, + "getActiveExperimentID should return the active experiment2"); + + list = yield experiments.getExperiments(); + Assert.equal(list.length, 2, "Experiment list should have 2 entries now."); + addons = yield getExperimentAddons(); + Assert.equal(addons.length, 1, "An experiment add-on is installed."); + + experimentListData[0].active = true; + experimentListData[0].endDate = now.getTime() + 10 * MS_IN_ONE_DAY; + for (let i = 0; i < experimentListData.length; ++i) { + let entry = experimentListData[i]; + for (let k of Object.keys(entry)) { + Assert.equal(entry[k], list[i][k], + "Entry " + i + " - Property '" + k + "' should match reference data."); + } + } + + Assert.equal(gTimerScheduleOffset, 10 * MS_IN_ONE_DAY, + "Experiment re-evaluation should have been scheduled correctly."); + + // Trigger update, clock set for experiment 2 to stop. + + now = futureDate(startDate2, 10 * MS_IN_ONE_DAY + 1000); + gTimerScheduleOffset = -1; + defineNow(gPolicy, now); + yield experiments.notify(); + Assert.equal(observerFireCount, ++expectedObserverFireCount, + "Experiments observer should have been called."); + + Assert.equal(experiments.getActiveExperimentID(), null, + "getActiveExperimentID should return null again2"); + + list = yield experiments.getExperiments(); + Assert.equal(list.length, 2, "Experiment list should have 2 entries now."); + addons = yield getExperimentAddons(); + Assert.equal(addons.length, 0, "No experiments add-ons are installed."); + + experimentListData[0].active = false; + experimentListData[0].endDate = now.getTime(); + for (let i = 0; i < experimentListData.length; ++i) { + let entry = experimentListData[i]; + for (let k of Object.keys(entry)) { + Assert.equal(entry[k], list[i][k], + "Entry " + i + " - Property '" + k + "' should match reference data."); + } + } + + // Cleanup. + + Services.obs.removeObserver(observer, OBSERVER_TOPIC); + yield testCleanup(experiments); +}); + +add_task(function* test_getActiveExperimentID() { + // Check that getActiveExperimentID returns the correct result even + // after .uninit() + + // Dates the following tests are based on. + + let baseDate = new Date(2014, 5, 1, 12); + let startDate1 = futureDate(baseDate, 50 * MS_IN_ONE_DAY); + let endDate1 = futureDate(baseDate, 100 * MS_IN_ONE_DAY); + + gManifestObject = { + "version": 1, + experiments: [ + { + id: EXPERIMENT1_ID, + xpiURL: gDataRoot + EXPERIMENT1_XPI_NAME, + xpiHash: EXPERIMENT1_XPI_SHA1, + startTime: dateToSeconds(startDate1), + endTime: dateToSeconds(endDate1), + maxActiveSeconds: 10 * SEC_IN_ONE_DAY, + appName: ["XPCShell"], + channel: ["nightly"], + }, + ], + }; + + let now = futureDate(startDate1, 5 * MS_IN_ONE_DAY); + gTimerScheduleOffset = -1; + defineNow(gPolicy, now); + + let experiments = new Experiments.Experiments(gPolicy); + yield experiments.updateManifest(); + + Assert.equal(experiments.getActiveExperimentID(), EXPERIMENT1_ID, + "getActiveExperimentID should return the active experiment1"); + + yield promiseRestartManager(); + Assert.equal(experiments.getActiveExperimentID(), EXPERIMENT1_ID, + "getActiveExperimentID should return the active experiment1 after uninit()"); + + yield testCleanup(experiments); +}); + +// Test that we handle the experiments addon already being +// installed properly. +// We should just pave over them. + +add_task(function* test_addonAlreadyInstalled() { + const OBSERVER_TOPIC = "experiments-changed"; + let observerFireCount = 0; + let expectedObserverFireCount = 0; + let observer = () => ++observerFireCount; + Services.obs.addObserver(observer, OBSERVER_TOPIC, false); + + // Dates the following tests are based on. + + let baseDate = new Date(2014, 5, 1, 12); + let startDate = futureDate(baseDate, 100 * MS_IN_ONE_DAY); + let endDate = futureDate(baseDate, 10000 * MS_IN_ONE_DAY); + + // The manifest data we test with. + + gManifestObject = { + "version": 1, + experiments: [ + { + id: EXPERIMENT1_ID, + xpiURL: gDataRoot + EXPERIMENT1_XPI_NAME, + xpiHash: EXPERIMENT1_XPI_SHA1, + startTime: dateToSeconds(startDate), + endTime: dateToSeconds(endDate), + maxActiveSeconds: 10 * SEC_IN_ONE_DAY, + appName: ["XPCShell"], + channel: ["nightly"], + }, + ], + }; + + let experiments = new Experiments.Experiments(gPolicy); + + // Trigger update, clock set to before any activation. + + let now = baseDate; + defineNow(gPolicy, now); + yield experiments.updateManifest(); + Assert.equal(observerFireCount, ++expectedObserverFireCount, + "Experiments observer should have been called."); + let list = yield experiments.getExperiments(); + Assert.equal(list.length, 0, "Experiment list should be empty."); + + // Trigger update, clock set for the experiment to start. + + now = futureDate(startDate, 10 * MS_IN_ONE_DAY); + defineNow(gPolicy, now); + yield experiments.updateManifest(); + Assert.equal(observerFireCount, ++expectedObserverFireCount, + "Experiments observer should have been called."); + + list = yield experiments.getExperiments(); + list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "Experiment list should have 1 entry now."); + Assert.equal(list[0].id, EXPERIMENT1_ID, "Experiment 1 should be the sole entry."); + Assert.equal(list[0].active, true, "Experiment 1 should be active."); + + let addons = yield getExperimentAddons(); + Assert.equal(addons.length, 1, "1 add-on is installed."); + + // Install conflicting addon. + + yield AddonManagerTesting.installXPIFromURL(gDataRoot + EXPERIMENT1_XPI_NAME, EXPERIMENT1_XPI_SHA1); + addons = yield getExperimentAddons(); + Assert.equal(addons.length, 1, "1 add-on is installed."); + list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "Experiment list should still have 1 entry."); + Assert.equal(list[0].id, EXPERIMENT1_ID, "Experiment 1 should be the sole entry."); + Assert.equal(list[0].active, true, "Experiment 1 should be active."); + + // Cleanup. + + Services.obs.removeObserver(observer, OBSERVER_TOPIC); + yield testCleanup(experiments); +}); + +add_task(function* test_lastActiveToday() { + let experiments = new Experiments.Experiments(gPolicy); + + replaceExperiments(experiments, FAKE_EXPERIMENTS_1); + + let e = yield experiments.getExperiments(); + Assert.equal(e.length, 1, "Monkeypatch successful."); + Assert.equal(e[0].id, "id1", "ID looks sane"); + Assert.ok(e[0].active, "Experiment is active."); + + let lastActive = yield experiments.lastActiveToday(); + Assert.equal(e[0], lastActive, "Last active object is expected."); + + replaceExperiments(experiments, FAKE_EXPERIMENTS_2); + e = yield experiments.getExperiments(); + Assert.equal(e.length, 2, "Monkeypatch successful."); + + defineNow(gPolicy, e[0].endDate); + + lastActive = yield experiments.lastActiveToday(); + Assert.ok(lastActive, "Have a last active experiment"); + Assert.equal(lastActive, e[0], "Last active object is expected."); + + yield testCleanup(experiments); +}); + +// Test explicitly disabling experiments. + +add_task(function* test_disableExperiment() { + // Dates this test is based on. + + let startDate = new Date(2004, 10, 9, 12); + let endDate = futureDate(startDate, 100 * MS_IN_ONE_DAY); + + // The manifest data we test with. + + gManifestObject = { + "version": 1, + experiments: [ + { + id: EXPERIMENT1_ID, + xpiURL: gDataRoot + EXPERIMENT1_XPI_NAME, + xpiHash: EXPERIMENT1_XPI_SHA1, + startTime: dateToSeconds(startDate), + endTime: dateToSeconds(endDate), + maxActiveSeconds: 10 * SEC_IN_ONE_DAY, + appName: ["XPCShell"], + channel: ["nightly"], + }, + ], + }; + + // Data to compare the result of Experiments.getExperiments() against. + + let experimentInfo = { + id: EXPERIMENT1_ID, + name: EXPERIMENT1_NAME, + description: "Yet another experiment that experiments experimentally.", + }; + + let experiments = new Experiments.Experiments(gPolicy); + + // Trigger update, clock set for the experiment to start. + + let now = futureDate(startDate, 5 * MS_IN_ONE_DAY); + defineNow(gPolicy, now); + yield experiments.updateManifest(); + + let list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "Experiment list should have 1 entry now."); + + experimentInfo.active = true; + experimentInfo.endDate = now.getTime() + 10 * MS_IN_ONE_DAY; + for (let k of Object.keys(experimentInfo)) { + Assert.equal(experimentInfo[k], list[0][k], + "Property " + k + " should match reference data."); + } + + // Test disabling the experiment. + + now = futureDate(now, 1 * MS_IN_ONE_DAY); + defineNow(gPolicy, now); + yield experiments.disableExperiment("foo"); + + list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "Experiment list should have 1 entry."); + + experimentInfo.active = false; + experimentInfo.endDate = now.getTime(); + for (let k of Object.keys(experimentInfo)) { + Assert.equal(experimentInfo[k], list[0][k], + "Property " + k + " should match reference data."); + } + + // Test that updating the list doesn't re-enable it. + + now = futureDate(now, 1 * MS_IN_ONE_DAY); + defineNow(gPolicy, now); + yield experiments.updateManifest(); + + list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "Experiment list should have 1 entry."); + + for (let k of Object.keys(experimentInfo)) { + Assert.equal(experimentInfo[k], list[0][k], + "Property " + k + " should match reference data."); + } + + yield testCleanup(experiments); +}); + +add_task(function* test_disableExperimentsFeature() { + // Dates this test is based on. + + let startDate = new Date(2004, 10, 9, 12); + let endDate = futureDate(startDate, 100 * MS_IN_ONE_DAY); + + // The manifest data we test with. + + gManifestObject = { + "version": 1, + experiments: [ + { + id: EXPERIMENT1_ID, + xpiURL: gDataRoot + EXPERIMENT1_XPI_NAME, + xpiHash: EXPERIMENT1_XPI_SHA1, + startTime: dateToSeconds(startDate), + endTime: dateToSeconds(endDate), + maxActiveSeconds: 10 * SEC_IN_ONE_DAY, + appName: ["XPCShell"], + channel: ["nightly"], + }, + ], + }; + + // Data to compare the result of Experiments.getExperiments() against. + + let experimentInfo = { + id: EXPERIMENT1_ID, + name: EXPERIMENT1_NAME, + description: "Yet another experiment that experiments experimentally.", + }; + + let experiments = new Experiments.Experiments(gPolicy); + Assert.equal(experiments.enabled, true, "Experiments feature should be enabled."); + + // Trigger update, clock set for the experiment to start. + + let now = futureDate(startDate, 5 * MS_IN_ONE_DAY); + defineNow(gPolicy, now); + yield experiments.updateManifest(); + + let list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "Experiment list should have 1 entry now."); + + experimentInfo.active = true; + experimentInfo.endDate = now.getTime() + 10 * MS_IN_ONE_DAY; + for (let k of Object.keys(experimentInfo)) { + Assert.equal(experimentInfo[k], list[0][k], + "Property " + k + " should match reference data."); + } + + // Test disabling experiments. + + experiments._toggleExperimentsEnabled(false); + yield experiments.notify(); + Assert.equal(experiments.enabled, false, "Experiments feature should be disabled now."); + + list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "Experiment list should have 1 entry."); + + experimentInfo.active = false; + experimentInfo.endDate = now.getTime(); + for (let k of Object.keys(experimentInfo)) { + Assert.equal(experimentInfo[k], list[0][k], + "Property " + k + " should match reference data."); + } + + // Test that updating the list doesn't re-enable it. + + now = futureDate(now, 1 * MS_IN_ONE_DAY); + defineNow(gPolicy, now); + try { + yield experiments.updateManifest(); + } catch (e) { + // Exception expected, the feature is disabled. + } + + list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "Experiment list should have 1 entry."); + + for (let k of Object.keys(experimentInfo)) { + Assert.equal(experimentInfo[k], list[0][k], + "Property " + k + " should match reference data."); + } + + yield testCleanup(experiments); +}); + +// Test that after a failed experiment install: +// * the next applicable experiment gets installed +// * changing the experiments data later triggers re-evaluation + +add_task(function* test_installFailure() { + const OBSERVER_TOPIC = "experiments-changed"; + let observerFireCount = 0; + let expectedObserverFireCount = 0; + let observer = () => ++observerFireCount; + Services.obs.addObserver(observer, OBSERVER_TOPIC, false); + + // Dates the following tests are based on. + + let baseDate = new Date(2014, 5, 1, 12); + let startDate = futureDate(baseDate, 100 * MS_IN_ONE_DAY); + let endDate = futureDate(baseDate, 10000 * MS_IN_ONE_DAY); + + // The manifest data we test with. + + gManifestObject = { + "version": 1, + experiments: [ + { + id: EXPERIMENT1_ID, + xpiURL: gDataRoot + EXPERIMENT1_XPI_NAME, + xpiHash: EXPERIMENT1_XPI_SHA1, + startTime: dateToSeconds(startDate), + endTime: dateToSeconds(endDate), + maxActiveSeconds: 10 * SEC_IN_ONE_DAY, + appName: ["XPCShell"], + channel: ["nightly"], + }, + { + id: EXPERIMENT2_ID, + xpiURL: gDataRoot + EXPERIMENT2_XPI_NAME, + xpiHash: EXPERIMENT2_XPI_SHA1, + startTime: dateToSeconds(startDate), + endTime: dateToSeconds(endDate), + maxActiveSeconds: 10 * SEC_IN_ONE_DAY, + appName: ["XPCShell"], + channel: ["nightly"], + }, + ], + }; + + // Data to compare the result of Experiments.getExperiments() against. + + let experimentListData = [ + { + id: EXPERIMENT1_ID, + name: EXPERIMENT1_NAME, + description: "Yet another experiment that experiments experimentally.", + }, + { + id: EXPERIMENT2_ID, + name: "Test experiment 2", + description: "And yet another experiment that experiments experimentally.", + }, + ]; + + let experiments = new Experiments.Experiments(gPolicy); + + // Trigger update, clock set to before any activation. + + let now = baseDate; + defineNow(gPolicy, now); + yield experiments.updateManifest(); + Assert.equal(observerFireCount, ++expectedObserverFireCount, + "Experiments observer should have been called."); + let list = yield experiments.getExperiments(); + Assert.equal(list.length, 0, "Experiment list should be empty."); + + // Trigger update, clock set for experiment 1 & 2 to start, + // invalid hash for experiment 1. + // Order in the manifest matters, so we should start experiment 1, + // fail to install it & start experiment 2 instead. + + now = futureDate(startDate, 10 * MS_IN_ONE_DAY); + defineNow(gPolicy, now); + gManifestObject.experiments[0].xpiHash = "sha1:0000000000000000000000000000000000000000"; + yield experiments.updateManifest(); + Assert.equal(observerFireCount, ++expectedObserverFireCount, + "Experiments observer should have been called."); + + list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "Experiment list should have 1 entry now."); + Assert.equal(list[0].id, EXPERIMENT2_ID, "Experiment 2 should be the sole entry."); + Assert.equal(list[0].active, true, "Experiment 2 should be active."); + + // Trigger update, clock set for experiment 2 to stop. + + now = futureDate(now, 20 * MS_IN_ONE_DAY); + defineNow(gPolicy, now); + yield experiments.updateManifest(); + Assert.equal(observerFireCount, ++expectedObserverFireCount, + "Experiments observer should have been called."); + + experimentListData[0].active = false; + experimentListData[0].endDate = now; + + list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "Experiment list should have 1 entry now."); + Assert.equal(list[0].id, EXPERIMENT2_ID, "Experiment 2 should be the sole entry."); + Assert.equal(list[0].active, false, "Experiment should not be active."); + + // Trigger update with a fixed entry for experiment 1, + // which should get re-evaluated & started now. + + now = futureDate(now, 20 * MS_IN_ONE_DAY); + defineNow(gPolicy, now); + gManifestObject.experiments[0].xpiHash = EXPERIMENT1_XPI_SHA1; + yield experiments.updateManifest(); + Assert.equal(observerFireCount, ++expectedObserverFireCount, + "Experiments observer should have been called."); + + experimentListData[0].active = true; + experimentListData[0].endDate = now.getTime() + 10 * MS_IN_ONE_DAY; + + list = yield experiments.getExperiments(); + Assert.equal(list.length, 2, "Experiment list should have 2 entries now."); + + for (let i = 0; i < experimentListData.length; ++i) { + let entry = experimentListData[i]; + for (let k of Object.keys(entry)) { + Assert.equal(entry[k], list[i][k], + "Entry " + i + " - Property '" + k + "' should match reference data."); + } + } + + yield testCleanup(experiments); +}); + +// Test that after an experiment was disabled by user action, +// the experiment is not activated again if manifest data changes. + +add_task(function* test_userDisabledAndUpdated() { + const OBSERVER_TOPIC = "experiments-changed"; + let observerFireCount = 0; + let expectedObserverFireCount = 0; + let observer = () => ++observerFireCount; + Services.obs.addObserver(observer, OBSERVER_TOPIC, false); + + // Dates the following tests are based on. + + let baseDate = new Date(2014, 5, 1, 12); + let startDate = futureDate(baseDate, 100 * MS_IN_ONE_DAY); + let endDate = futureDate(baseDate, 10000 * MS_IN_ONE_DAY); + + // The manifest data we test with. + + gManifestObject = { + "version": 1, + experiments: [ + { + id: EXPERIMENT1_ID, + xpiURL: gDataRoot + EXPERIMENT1_XPI_NAME, + xpiHash: EXPERIMENT1_XPI_SHA1, + startTime: dateToSeconds(startDate), + endTime: dateToSeconds(endDate), + maxActiveSeconds: 10 * SEC_IN_ONE_DAY, + appName: ["XPCShell"], + channel: ["nightly"], + }, + ], + }; + + let experiments = new Experiments.Experiments(gPolicy); + + // Trigger update, clock set to before any activation. + + let now = baseDate; + defineNow(gPolicy, now); + yield experiments.updateManifest(); + Assert.equal(observerFireCount, ++expectedObserverFireCount, + "Experiments observer should have been called."); + let list = yield experiments.getExperiments(); + Assert.equal(list.length, 0, "Experiment list should be empty."); + + // Trigger update, clock set for experiment 1 to start. + + now = futureDate(startDate, 10 * MS_IN_ONE_DAY); + defineNow(gPolicy, now); + yield experiments.updateManifest(); + Assert.equal(observerFireCount, ++expectedObserverFireCount, + "Experiments observer should have been called."); + + list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "Experiment list should have 1 entry now."); + Assert.equal(list[0].id, EXPERIMENT1_ID, "Experiment 1 should be the sole entry."); + Assert.equal(list[0].active, true, "Experiment 1 should be active."); + let todayActive = yield experiments.lastActiveToday(); + Assert.ok(todayActive, "Last active for today reports a value."); + Assert.equal(todayActive.id, list[0].id, "The entry is what we expect."); + + // Explicitly disable an experiment. + + now = futureDate(now, 20 * MS_IN_ONE_DAY); + defineNow(gPolicy, now); + yield experiments.disableExperiment("foo"); + Assert.equal(observerFireCount, ++expectedObserverFireCount, + "Experiments observer should have been called."); + + list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "Experiment list should have 1 entry now."); + Assert.equal(list[0].id, EXPERIMENT1_ID, "Experiment 1 should be the sole entry."); + Assert.equal(list[0].active, false, "Experiment should not be active anymore."); + todayActive = yield experiments.lastActiveToday(); + Assert.ok(todayActive, "Last active for today still returns a value."); + Assert.equal(todayActive.id, list[0].id, "The ID is still the same."); + + // Trigger an update with a faked change for experiment 1. + + now = futureDate(now, 20 * MS_IN_ONE_DAY); + defineNow(gPolicy, now); + experiments._experiments.get(EXPERIMENT1_ID)._manifestData.xpiHash = + "sha1:0000000000000000000000000000000000000000"; + yield experiments.updateManifest(); + Assert.equal(observerFireCount, expectedObserverFireCount, + "Experiments observer should not have been called."); + + list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "Experiment list should have 1 entry now."); + Assert.equal(list[0].id, EXPERIMENT1_ID, "Experiment 1 should be the sole entry."); + Assert.equal(list[0].active, false, "Experiment should still be inactive."); + + // Cleanup. + + Services.obs.removeObserver(observer, OBSERVER_TOPIC); + yield testCleanup(experiments); +}); + +// Test that changing the hash for an active experiments triggers an +// update for it. + +add_task(function* test_updateActiveExperiment() { + const OBSERVER_TOPIC = "experiments-changed"; + let observerFireCount = 0; + let expectedObserverFireCount = 0; + let observer = () => ++observerFireCount; + Services.obs.addObserver(observer, OBSERVER_TOPIC, false); + + // Dates the following tests are based on. + + let baseDate = new Date(2014, 5, 1, 12); + let startDate = futureDate(baseDate, 100 * MS_IN_ONE_DAY); + let endDate = futureDate(baseDate, 10000 * MS_IN_ONE_DAY); + + // The manifest data we test with. + + gManifestObject = { + "version": 1, + experiments: [ + { + id: EXPERIMENT1_ID, + xpiURL: gDataRoot + EXPERIMENT1_XPI_NAME, + xpiHash: EXPERIMENT1_XPI_SHA1, + startTime: dateToSeconds(startDate), + endTime: dateToSeconds(endDate), + maxActiveSeconds: 10 * SEC_IN_ONE_DAY, + appName: ["XPCShell"], + channel: ["nightly"], + }, + ], + }; + + let experiments = new Experiments.Experiments(gPolicy); + + // Trigger update, clock set to before any activation. + + let now = baseDate; + defineNow(gPolicy, now); + yield experiments.updateManifest(); + Assert.equal(observerFireCount, ++expectedObserverFireCount, + "Experiments observer should have been called."); + let list = yield experiments.getExperiments(); + Assert.equal(list.length, 0, "Experiment list should be empty."); + + let todayActive = yield experiments.lastActiveToday(); + Assert.equal(todayActive, null, "No experiment active today."); + + // Trigger update, clock set for the experiment to start. + + now = futureDate(startDate, 10 * MS_IN_ONE_DAY); + defineNow(gPolicy, now); + yield experiments.updateManifest(); + Assert.equal(observerFireCount, ++expectedObserverFireCount, + "Experiments observer should have been called."); + + list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "Experiment list should have 1 entry now."); + Assert.equal(list[0].id, EXPERIMENT1_ID, "Experiment 1 should be the sole entry."); + Assert.equal(list[0].active, true, "Experiment 1 should be active."); + Assert.equal(list[0].name, EXPERIMENT1_NAME, "Experiments name should match."); + todayActive = yield experiments.lastActiveToday(); + Assert.ok(todayActive, "todayActive() returns a value."); + Assert.equal(todayActive.id, list[0].id, "It returns the active experiment."); + + // Trigger an update for the active experiment by changing it's hash (and xpi) + // in the manifest. + + now = futureDate(now, 1 * MS_IN_ONE_DAY); + defineNow(gPolicy, now); + gManifestObject.experiments[0].xpiHash = EXPERIMENT1A_XPI_SHA1; + gManifestObject.experiments[0].xpiURL = gDataRoot + EXPERIMENT1A_XPI_NAME; + yield experiments.updateManifest(); + Assert.equal(observerFireCount, ++expectedObserverFireCount, + "Experiments observer should have been called."); + + list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "Experiment list should have 1 entry now."); + Assert.equal(list[0].id, EXPERIMENT1_ID, "Experiment 1 should be the sole entry."); + Assert.equal(list[0].active, true, "Experiment 1 should still be active."); + Assert.equal(list[0].name, EXPERIMENT1A_NAME, "Experiments name should have been updated."); + todayActive = yield experiments.lastActiveToday(); + Assert.equal(todayActive.id, list[0].id, "last active today is still sane."); + + // Cleanup. + + Services.obs.removeObserver(observer, OBSERVER_TOPIC); + yield testCleanup(experiments); +}); + +// Tests that setting the disable flag for an active experiment +// stops it. + +add_task(function* test_disableActiveExperiment() { + const OBSERVER_TOPIC = "experiments-changed"; + let observerFireCount = 0; + let expectedObserverFireCount = 0; + let observer = () => ++observerFireCount; + Services.obs.addObserver(observer, OBSERVER_TOPIC, false); + + // Dates the following tests are based on. + + let baseDate = new Date(2014, 5, 1, 12); + let startDate = futureDate(baseDate, 100 * MS_IN_ONE_DAY); + let endDate = futureDate(baseDate, 10000 * MS_IN_ONE_DAY); + + // The manifest data we test with. + + gManifestObject = { + "version": 1, + experiments: [ + { + id: EXPERIMENT1_ID, + xpiURL: gDataRoot + EXPERIMENT1_XPI_NAME, + xpiHash: EXPERIMENT1_XPI_SHA1, + startTime: dateToSeconds(startDate), + endTime: dateToSeconds(endDate), + maxActiveSeconds: 10 * SEC_IN_ONE_DAY, + appName: ["XPCShell"], + channel: ["nightly"], + }, + ], + }; + + let experiments = new Experiments.Experiments(gPolicy); + + // Trigger update, clock set to before any activation. + + let now = baseDate; + defineNow(gPolicy, now); + yield experiments.updateManifest(); + Assert.equal(observerFireCount, ++expectedObserverFireCount, + "Experiments observer should have been called."); + let list = yield experiments.getExperiments(); + Assert.equal(list.length, 0, "Experiment list should be empty."); + + // Trigger update, clock set for the experiment to start. + + now = futureDate(startDate, 10 * MS_IN_ONE_DAY); + defineNow(gPolicy, now); + yield experiments.updateManifest(); + Assert.equal(observerFireCount, ++expectedObserverFireCount, + "Experiments observer should have been called."); + + list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "Experiment list should have 1 entry now."); + Assert.equal(list[0].id, EXPERIMENT1_ID, "Experiment 1 should be the sole entry."); + Assert.equal(list[0].active, true, "Experiment 1 should be active."); + + // Trigger an update with the experiment being disabled. + + now = futureDate(now, 1 * MS_IN_ONE_DAY); + defineNow(gPolicy, now); + gManifestObject.experiments[0].disabled = true; + yield experiments.updateManifest(); + Assert.equal(observerFireCount, ++expectedObserverFireCount, + "Experiments observer should have been called."); + + list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "Experiment list should have 1 entry now."); + Assert.equal(list[0].id, EXPERIMENT1_ID, "Experiment 1 should be the sole entry."); + Assert.equal(list[0].active, false, "Experiment 1 should be disabled."); + + // Check that the experiment stays disabled. + + now = futureDate(now, 1 * MS_IN_ONE_DAY); + defineNow(gPolicy, now); + delete gManifestObject.experiments[0].disabled; + yield experiments.updateManifest(); + + list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "Experiment list should have 1 entry now."); + Assert.equal(list[0].id, EXPERIMENT1_ID, "Experiment 1 should be the sole entry."); + Assert.equal(list[0].active, false, "Experiment 1 should still be disabled."); + + // Cleanup. + + Services.obs.removeObserver(observer, OBSERVER_TOPIC); + yield testCleanup(experiments); +}); + +// Test that: +// * setting the frozen flag for a not-yet-started experiment keeps +// it from starting +// * after a removing the frozen flag, the experiment can still start + +add_task(function* test_freezePendingExperiment() { + const OBSERVER_TOPIC = "experiments-changed"; + let observerFireCount = 0; + let expectedObserverFireCount = 0; + let observer = () => ++observerFireCount; + Services.obs.addObserver(observer, OBSERVER_TOPIC, false); + + // Dates the following tests are based on. + + let baseDate = new Date(2014, 5, 1, 12); + let startDate = futureDate(baseDate, 100 * MS_IN_ONE_DAY); + let endDate = futureDate(baseDate, 10000 * MS_IN_ONE_DAY); + + // The manifest data we test with. + + gManifestObject = { + "version": 1, + experiments: [ + { + id: EXPERIMENT1_ID, + xpiURL: gDataRoot + EXPERIMENT1_XPI_NAME, + xpiHash: EXPERIMENT1_XPI_SHA1, + startTime: dateToSeconds(startDate), + endTime: dateToSeconds(endDate), + maxActiveSeconds: 10 * SEC_IN_ONE_DAY, + appName: ["XPCShell"], + channel: ["nightly"], + }, + ], + }; + + let experiments = new Experiments.Experiments(gPolicy); + + // Trigger update, clock set to before any activation. + + let now = baseDate; + defineNow(gPolicy, now); + yield experiments.updateManifest(); + Assert.equal(observerFireCount, ++expectedObserverFireCount, + "Experiments observer should have been called."); + let list = yield experiments.getExperiments(); + Assert.equal(list.length, 0, "Experiment list should be empty."); + + // Trigger update, clock set for the experiment to start but frozen. + + now = futureDate(startDate, 10 * MS_IN_ONE_DAY); + defineNow(gPolicy, now); + gManifestObject.experiments[0].frozen = true; + yield experiments.updateManifest(); + Assert.equal(observerFireCount, expectedObserverFireCount, + "Experiments observer should have not been called."); + + list = yield experiments.getExperiments(); + Assert.equal(list.length, 0, "Experiment list should have no entries yet."); + + // Trigger an update with the experiment not being frozen anymore. + + now = futureDate(now, 1 * MS_IN_ONE_DAY); + defineNow(gPolicy, now); + delete gManifestObject.experiments[0].frozen; + yield experiments.updateManifest(); + Assert.equal(observerFireCount, ++expectedObserverFireCount, + "Experiments observer should have been called."); + + list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "Experiment list should have 1 entry now."); + Assert.equal(list[0].id, EXPERIMENT1_ID, "Experiment 1 should be the sole entry."); + Assert.equal(list[0].active, true, "Experiment 1 should be active now."); + + // Cleanup. + + Services.obs.removeObserver(observer, OBSERVER_TOPIC); + yield testCleanup(experiments); +}); + +// Test that setting the frozen flag for an active experiment doesn't +// stop it. + +add_task(function* test_freezeActiveExperiment() { + const OBSERVER_TOPIC = "experiments-changed"; + let observerFireCount = 0; + let expectedObserverFireCount = 0; + let observer = () => ++observerFireCount; + Services.obs.addObserver(observer, OBSERVER_TOPIC, false); + + // Dates the following tests are based on. + + let baseDate = new Date(2014, 5, 1, 12); + let startDate = futureDate(baseDate, 100 * MS_IN_ONE_DAY); + let endDate = futureDate(baseDate, 10000 * MS_IN_ONE_DAY); + + // The manifest data we test with. + + gManifestObject = { + "version": 1, + experiments: [ + { + id: EXPERIMENT1_ID, + xpiURL: gDataRoot + EXPERIMENT1_XPI_NAME, + xpiHash: EXPERIMENT1_XPI_SHA1, + startTime: dateToSeconds(startDate), + endTime: dateToSeconds(endDate), + maxActiveSeconds: 10 * SEC_IN_ONE_DAY, + appName: ["XPCShell"], + channel: ["nightly"], + }, + ], + }; + + let experiments = new Experiments.Experiments(gPolicy); + + // Trigger update, clock set to before any activation. + + let now = baseDate; + defineNow(gPolicy, now); + yield experiments.updateManifest(); + Assert.equal(observerFireCount, ++expectedObserverFireCount, + "Experiments observer should have been called."); + let list = yield experiments.getExperiments(); + Assert.equal(list.length, 0, "Experiment list should be empty."); + + // Trigger update, clock set for the experiment to start. + + now = futureDate(startDate, 10 * MS_IN_ONE_DAY); + defineNow(gPolicy, now); + yield experiments.updateManifest(); + Assert.equal(observerFireCount, ++expectedObserverFireCount, + "Experiments observer should have been called."); + + list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "Experiment list should have 1 entry now."); + Assert.equal(list[0].id, EXPERIMENT1_ID, "Experiment 1 should be the sole entry."); + Assert.equal(list[0].active, true, "Experiment 1 should be active."); + Assert.equal(list[0].name, EXPERIMENT1_NAME, "Experiments name should match."); + + // Trigger an update with the experiment being disabled. + + now = futureDate(now, 1 * MS_IN_ONE_DAY); + defineNow(gPolicy, now); + gManifestObject.experiments[0].frozen = true; + yield experiments.updateManifest(); + Assert.equal(observerFireCount, expectedObserverFireCount, + "Experiments observer should have been called."); + + list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "Experiment list should have 1 entry now."); + Assert.equal(list[0].id, EXPERIMENT1_ID, "Experiment 1 should be the sole entry."); + Assert.equal(list[0].active, true, "Experiment 1 should still be active."); + + // Cleanup. + + Services.obs.removeObserver(observer, OBSERVER_TOPIC); + yield testCleanup(experiments); +}); + +// Test that removing an active experiment from the manifest doesn't +// stop it. + +add_task(function* test_removeActiveExperiment() { + const OBSERVER_TOPIC = "experiments-changed"; + let observerFireCount = 0; + let expectedObserverFireCount = 0; + let observer = () => ++observerFireCount; + Services.obs.addObserver(observer, OBSERVER_TOPIC, false); + + // Dates the following tests are based on. + + let baseDate = new Date(2014, 5, 1, 12); + let startDate = futureDate(baseDate, 100 * MS_IN_ONE_DAY); + let endDate = futureDate(baseDate, 10000 * MS_IN_ONE_DAY); + let startDate2 = futureDate(baseDate, 20000 * MS_IN_ONE_DAY); + let endDate2 = futureDate(baseDate, 30000 * MS_IN_ONE_DAY); + + // The manifest data we test with. + + gManifestObject = { + "version": 1, + experiments: [ + { + id: EXPERIMENT1_ID, + xpiURL: gDataRoot + EXPERIMENT1_XPI_NAME, + xpiHash: EXPERIMENT1_XPI_SHA1, + startTime: dateToSeconds(startDate), + endTime: dateToSeconds(endDate), + maxActiveSeconds: 10 * SEC_IN_ONE_DAY, + appName: ["XPCShell"], + channel: ["nightly"], + }, + { + id: EXPERIMENT2_ID, + xpiURL: gDataRoot + EXPERIMENT1_XPI_NAME, + xpiHash: EXPERIMENT2_XPI_SHA1, + startTime: dateToSeconds(startDate2), + endTime: dateToSeconds(endDate2), + maxActiveSeconds: 10 * SEC_IN_ONE_DAY, + appName: ["XPCShell"], + channel: ["nightly"], + }, + ], + }; + + let experiments = new Experiments.Experiments(gPolicy); + + // Trigger update, clock set to before any activation. + + let now = baseDate; + defineNow(gPolicy, now); + yield experiments.updateManifest(); + Assert.equal(observerFireCount, ++expectedObserverFireCount, + "Experiments observer should have been called."); + let list = yield experiments.getExperiments(); + Assert.equal(list.length, 0, "Experiment list should be empty."); + + // Trigger update, clock set for the experiment to start. + + now = futureDate(startDate, 10 * MS_IN_ONE_DAY); + defineNow(gPolicy, now); + yield experiments.updateManifest(); + Assert.equal(observerFireCount, ++expectedObserverFireCount, + "Experiments observer should have been called."); + + list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "Experiment list should have 1 entry now."); + Assert.equal(list[0].id, EXPERIMENT1_ID, "Experiment 1 should be the sole entry."); + Assert.equal(list[0].active, true, "Experiment 1 should be active."); + Assert.equal(list[0].name, EXPERIMENT1_NAME, "Experiments name should match."); + + // Trigger an update with experiment 1 missing from the manifest + + now = futureDate(now, 1 * MS_IN_ONE_DAY); + defineNow(gPolicy, now); + gManifestObject.experiments[0].frozen = true; + yield experiments.updateManifest(); + Assert.equal(observerFireCount, expectedObserverFireCount, + "Experiments observer should have been called."); + + list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "Experiment list should have 1 entry now."); + Assert.equal(list[0].id, EXPERIMENT1_ID, "Experiment 1 should be the sole entry."); + Assert.equal(list[0].active, true, "Experiment 1 should still be active."); + + // Cleanup. + + Services.obs.removeObserver(observer, OBSERVER_TOPIC); + yield testCleanup(experiments); +}); + +// Test that we correctly handle experiment start & install failures. + +add_task(function* test_invalidUrl() { + const OBSERVER_TOPIC = "experiments-changed"; + let observerFireCount = 0; + let expectedObserverFireCount = 0; + let observer = () => ++observerFireCount; + Services.obs.addObserver(observer, OBSERVER_TOPIC, false); + + // Dates the following tests are based on. + + let baseDate = new Date(2014, 5, 1, 12); + let startDate = futureDate(baseDate, 100 * MS_IN_ONE_DAY); + let endDate = futureDate(baseDate, 10000 * MS_IN_ONE_DAY); + + // The manifest data we test with. + + gManifestObject = { + "version": 1, + experiments: [ + { + id: EXPERIMENT1_ID, + xpiURL: gDataRoot + EXPERIMENT1_XPI_NAME + ".invalid", + xpiHash: EXPERIMENT1_XPI_SHA1, + startTime: 0, + endTime: dateToSeconds(endDate), + maxActiveSeconds: 10 * SEC_IN_ONE_DAY, + appName: ["XPCShell"], + channel: ["nightly"], + }, + ], + }; + + let experiments = new Experiments.Experiments(gPolicy); + + // Trigger update, clock set for the experiment to start. + + let now = futureDate(startDate, 10 * MS_IN_ONE_DAY); + defineNow(gPolicy, now); + gTimerScheduleOffset = null; + + yield experiments.updateManifest(); + Assert.equal(observerFireCount, ++expectedObserverFireCount, + "Experiments observer should have been called."); + Assert.equal(gTimerScheduleOffset, null, "No new timer should have been scheduled."); + + let list = yield experiments.getExperiments(); + Assert.equal(list.length, 0, "Experiment list should be empty."); + + // Cleanup. + + Services.obs.removeObserver(observer, OBSERVER_TOPIC); + yield testCleanup(experiments); +}); + +// Test that we handle it properly when active experiment addons are being +// uninstalled. + +add_task(function* test_unexpectedUninstall() { + const OBSERVER_TOPIC = "experiments-changed"; + let observerFireCount = 0; + let expectedObserverFireCount = 0; + let observer = () => ++observerFireCount; + Services.obs.addObserver(observer, OBSERVER_TOPIC, false); + + // Dates the following tests are based on. + + let baseDate = new Date(2014, 5, 1, 12); + let startDate = futureDate(baseDate, 100 * MS_IN_ONE_DAY); + let endDate = futureDate(baseDate, 10000 * MS_IN_ONE_DAY); + + // The manifest data we test with. + + gManifestObject = { + "version": 1, + experiments: [ + { + id: EXPERIMENT1_ID, + xpiURL: gDataRoot + EXPERIMENT1_XPI_NAME, + xpiHash: EXPERIMENT1_XPI_SHA1, + startTime: dateToSeconds(startDate), + endTime: dateToSeconds(endDate), + maxActiveSeconds: 10 * SEC_IN_ONE_DAY, + appName: ["XPCShell"], + channel: ["nightly"], + }, + ], + }; + + let experiments = new Experiments.Experiments(gPolicy); + + // Trigger update, clock set to before any activation. + + let now = baseDate; + defineNow(gPolicy, now); + yield experiments.updateManifest(); + Assert.equal(observerFireCount, ++expectedObserverFireCount, + "Experiments observer should have been called."); + let list = yield experiments.getExperiments(); + Assert.equal(list.length, 0, "Experiment list should be empty."); + + // Trigger update, clock set for the experiment to start. + + now = futureDate(startDate, 10 * MS_IN_ONE_DAY); + defineNow(gPolicy, now); + yield experiments.updateManifest(); + Assert.equal(observerFireCount, ++expectedObserverFireCount, + "Experiments observer should have been called."); + + list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "Experiment list should have 1 entry now."); + Assert.equal(list[0].id, EXPERIMENT1_ID, "Experiment 1 should be the sole entry."); + Assert.equal(list[0].active, true, "Experiment 1 should be active."); + + // Uninstall the addon through the addon manager instead of stopping it through + // the experiments API. + + yield AddonManagerTesting.uninstallAddonByID(EXPERIMENT1_ID); + yield experiments._mainTask; + + yield experiments.notify(); + + list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "Experiment list should have 1 entry now."); + Assert.equal(list[0].id, EXPERIMENT1_ID, "Experiment 1 should be the sole entry."); + Assert.equal(list[0].active, false, "Experiment 1 should not be active anymore."); + + // Cleanup. + + Services.obs.removeObserver(observer, OBSERVER_TOPIC); + yield testCleanup(experiments); +}); + +// If the Addon Manager knows of an experiment that we don't, it should get +// uninstalled. +add_task(function* testUnknownExperimentsUninstalled() { + let experiments = new Experiments.Experiments(gPolicy); + + let addons = yield getExperimentAddons(); + Assert.equal(addons.length, 0, "Precondition: No experiment add-ons are present."); + + // Simulate us not listening. + experiments._unregisterWithAddonManager(); + yield AddonManagerTesting.installXPIFromURL(gDataRoot + EXPERIMENT1_XPI_NAME, EXPERIMENT1_XPI_SHA1); + experiments._registerWithAddonManager(); + + addons = yield getExperimentAddons(); + Assert.equal(addons.length, 1, "Experiment 1 installed via AddonManager"); + + // Simulate no known experiments. + gManifestObject = { + "version": 1, + experiments: [], + }; + + yield experiments.updateManifest(); + let fromManifest = yield experiments.getExperiments(); + Assert.equal(fromManifest.length, 0, "No experiments known in manifest."); + + // And the unknown add-on should be gone. + addons = yield getExperimentAddons(); + Assert.equal(addons.length, 0, "Experiment 1 was uninstalled."); + + yield testCleanup(experiments); +}); + +// If someone else installs an experiment add-on, we detect and stop that. +add_task(function* testForeignExperimentInstall() { + let experiments = new Experiments.Experiments(gPolicy); + + gManifestObject = { + "version": 1, + experiments: [], + }; + + yield experiments.init(); + + let addons = yield getExperimentAddons(); + Assert.equal(addons.length, 0, "Precondition: No experiment add-ons present."); + + let failed = false; + try { + yield AddonManagerTesting.installXPIFromURL(gDataRoot + EXPERIMENT1_XPI_NAME, EXPERIMENT1_XPI_SHA1); + } catch (ex) { + failed = true; + } + Assert.ok(failed, "Add-on install should not have completed successfully"); + addons = yield getExperimentAddons(); + Assert.equal(addons.length, 0, "Add-on install should have been cancelled."); + + yield testCleanup(experiments); +}); + +// Experiment add-ons will be disabled after Addon Manager restarts. Ensure +// we enable them automatically. +add_task(function* testEnabledAfterRestart() { + let experiments = new Experiments.Experiments(gPolicy); + + gManifestObject = { + "version": 1, + experiments: [ + { + id: EXPERIMENT1_ID, + xpiURL: gDataRoot + EXPERIMENT1_XPI_NAME, + xpiHash: EXPERIMENT1_XPI_SHA1, + startTime: gPolicy.now().getTime() / 1000 - 60, + endTime: gPolicy.now().getTime() / 1000 + 60, + maxActiveSeconds: 10 * SEC_IN_ONE_DAY, + appName: ["XPCShell"], + channel: ["nightly"], + }, + ], + }; + + let addons = yield getExperimentAddons(); + Assert.equal(addons.length, 0, "Precondition: No experiment add-ons installed."); + + yield experiments.updateManifest(); + let fromManifest = yield experiments.getExperiments(); + Assert.equal(fromManifest.length, 1, "A single experiment is known."); + + addons = yield getExperimentAddons(); + Assert.equal(addons.length, 1, "A single experiment add-on is installed."); + Assert.ok(addons[0].isActive, "That experiment is active."); + + dump("Restarting Addon Manager\n"); + yield promiseRestartManager(); + experiments = new Experiments.Experiments(gPolicy); + + addons = yield getExperimentAddons(); + Assert.equal(addons.length, 1, "The experiment is still there after restart."); + Assert.ok(addons[0].userDisabled, "But it is disabled."); + Assert.equal(addons[0].isActive, false, "And not active."); + + yield experiments.updateManifest(); + Assert.ok(addons[0].isActive, "It activates when the manifest is evaluated."); + + yield testCleanup(experiments); +}); + +// If experiment add-ons were ever started, maxStartTime shouldn't be evaluated +// anymore. Ensure that if maxStartTime is passed but experiment has started +// already, maxStartTime does not cause deactivation. + +add_task(function* testMaxStartTimeEvaluation() { + + // Dates the following tests are based on. + + let startDate = new Date(2014, 5, 1, 12); + let now = futureDate(startDate, 10 * MS_IN_ONE_DAY); + let maxStartDate = futureDate(startDate, 100 * MS_IN_ONE_DAY); + let endDate = futureDate(startDate, 1000 * MS_IN_ONE_DAY); + + defineNow(gPolicy, now); + + // The manifest data we test with. + // We set a value for maxStartTime. + + gManifestObject = { + "version": 1, + experiments: [ + { + id: EXPERIMENT1_ID, + xpiURL: gDataRoot + EXPERIMENT1_XPI_NAME, + xpiHash: EXPERIMENT1_XPI_SHA1, + startTime: dateToSeconds(startDate), + endTime: dateToSeconds(endDate), + maxActiveSeconds: 1000 * SEC_IN_ONE_DAY, + maxStartTime: dateToSeconds(maxStartDate), + appName: ["XPCShell"], + channel: ["nightly"], + }, + ], + }; + + let experiments = new Experiments.Experiments(gPolicy); + + let addons = yield getExperimentAddons(); + Assert.equal(addons.length, 0, "Precondition: No experiment add-ons installed."); + + yield experiments.updateManifest(); + let fromManifest = yield experiments.getExperiments(); + Assert.equal(fromManifest.length, 1, "A single experiment is known."); + + addons = yield getExperimentAddons(); + Assert.equal(addons.length, 1, "A single experiment add-on is installed."); + Assert.ok(addons[0].isActive, "That experiment is active."); + + dump("Setting current time to maxStartTime + 100 days and reloading manifest\n"); + now = futureDate(maxStartDate, 100 * MS_IN_ONE_DAY); + defineNow(gPolicy, now); + yield experiments.updateManifest(); + + addons = yield getExperimentAddons(); + Assert.equal(addons.length, 1, "The experiment is still there."); + Assert.ok(addons[0].isActive, "It is still active."); + + yield testCleanup(experiments); +}); + +// Test coverage for an add-on uninstall disabling the experiment and that it stays +// disabled over restarts. +add_task(function* test_foreignUninstallAndRestart() { + let experiments = new Experiments.Experiments(gPolicy); + + gManifestObject = { + "version": 1, + experiments: [ + { + id: EXPERIMENT1_ID, + xpiURL: gDataRoot + EXPERIMENT1_XPI_NAME, + xpiHash: EXPERIMENT1_XPI_SHA1, + startTime: gPolicy.now().getTime() / 1000 - 60, + endTime: gPolicy.now().getTime() / 1000 + 60, + maxActiveSeconds: 10 * SEC_IN_ONE_DAY, + appName: ["XPCShell"], + channel: ["nightly"], + }, + ], + }; + + let addons = yield getExperimentAddons(); + Assert.equal(addons.length, 0, "Precondition: No experiment add-ons installed."); + + yield experiments.updateManifest(); + let experimentList = yield experiments.getExperiments(); + Assert.equal(experimentList.length, 1, "A single experiment is known."); + + addons = yield getExperimentAddons(); + Assert.equal(addons.length, 1, "A single experiment add-on is installed."); + Assert.ok(addons[0].isActive, "That experiment is active."); + + yield AddonManagerTesting.uninstallAddonByID(EXPERIMENT1_ID); + yield experiments._mainTask; + + addons = yield getExperimentAddons(); + Assert.equal(addons.length, 0, "Experiment add-on should have been removed."); + + experimentList = yield experiments.getExperiments(); + Assert.equal(experimentList.length, 1, "A single experiment is known."); + Assert.equal(experimentList[0].id, EXPERIMENT1_ID, "Experiment 1 should be the sole entry."); + Assert.ok(!experimentList[0].active, "Experiment 1 should not be active anymore."); + + // Fake restart behaviour. + yield promiseRestartManager(); + experiments = new Experiments.Experiments(gPolicy); + yield experiments.updateManifest(); + + addons = yield getExperimentAddons(); + Assert.equal(addons.length, 0, "No experiment add-ons installed."); + + experimentList = yield experiments.getExperiments(); + Assert.equal(experimentList.length, 1, "A single experiment is known."); + Assert.equal(experimentList[0].id, EXPERIMENT1_ID, "Experiment 1 should be the sole entry."); + Assert.ok(!experimentList[0].active, "Experiment 1 should not be active."); + + yield testCleanup(experiments); +}); diff --git a/application/basilisk/experiments/test/xpcshell/test_cache.js b/application/basilisk/experiments/test/xpcshell/test_cache.js new file mode 100644 index 000000000..622b4e401 --- /dev/null +++ b/application/basilisk/experiments/test/xpcshell/test_cache.js @@ -0,0 +1,399 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +Cu.import("resource://testing-common/httpd.js"); +XPCOMUtils.defineLazyModuleGetter(this, "Experiments", + "resource:///modules/experiments/Experiments.jsm"); + +const MANIFEST_HANDLER = "manifests/handler"; + +const SEC_IN_ONE_DAY = 24 * 60 * 60; +const MS_IN_ONE_DAY = SEC_IN_ONE_DAY * 1000; + +var gHttpServer = null; +var gHttpRoot = null; +var gDataRoot = null; +var gPolicy = null; +var gManifestObject = null; +var gManifestHandlerURI = null; + +function run_test() { + run_next_test(); +} + +add_task(function* test_setup() { + loadAddonManager(); + yield removeCacheFile(); + + gHttpServer = new HttpServer(); + gHttpServer.start(-1); + let port = gHttpServer.identity.primaryPort; + gHttpRoot = "http://localhost:" + port + "/"; + gDataRoot = gHttpRoot + "data/"; + gManifestHandlerURI = gHttpRoot + MANIFEST_HANDLER; + gHttpServer.registerDirectory("/data/", do_get_cwd()); + gHttpServer.registerPathHandler("/" + MANIFEST_HANDLER, (request, response) => { + response.setStatusLine(null, 200, "OK"); + response.write(JSON.stringify(gManifestObject)); + response.processAsync(); + response.finish(); + }); + do_register_cleanup(() => gHttpServer.stop(() => {})); + + Services.prefs.setBoolPref(PREF_EXPERIMENTS_ENABLED, true); + Services.prefs.setIntPref(PREF_LOGGING_LEVEL, 0); + Services.prefs.setBoolPref(PREF_LOGGING_DUMP, true); + Services.prefs.setCharPref(PREF_MANIFEST_URI, gManifestHandlerURI); + Services.prefs.setIntPref(PREF_FETCHINTERVAL, 0); + + gPolicy = new Experiments.Policy(); + patchPolicy(gPolicy, { + updatechannel: () => "nightly", + oneshotTimer: (callback, timeout, thisObj, name) => {}, + }); +}); + +function checkExperimentListsEqual(list, list2) { + Assert.equal(list.length, list2.length, "Lists should have the same length.") + + for (let i = 0; i < list.length; ++i) { + for (let k of Object.keys(list[i])) { + Assert.equal(list[i][k], list2[i][k], + "Field '" + k + "' should match for list entry " + i + "."); + } + } +} + +function checkExperimentSerializations(experimentEntryIterator) { + for (let experiment of experimentEntryIterator) { + let experiment2 = new Experiments.ExperimentEntry(gPolicy); + let jsonStr = JSON.stringify(experiment.toJSON()); + Assert.ok(experiment2.initFromCacheData(JSON.parse(jsonStr)), + "Should have initialized successfully from JSON serialization."); + Assert.equal(JSON.stringify(experiment), JSON.stringify(experiment2), + "Object stringifications should match."); + } +} + +function validateCache(cachedExperiments, experimentIds) { + let cachedExperimentIds = new Set(cachedExperiments); + Assert.equal(cachedExperimentIds.size, experimentIds.length, + "The number of cached experiments does not match with the provided list"); + for (let id of experimentIds) { + Assert.ok(cachedExperimentIds.has(id), "The cache must contain the experiment with id " + id); + } +} + +// Set up an experiments instance and check if it is properly restored from cache. + +add_task(function* test_cache() { + // The manifest data we test with. + + gManifestObject = { + "version": 1, + experiments: [ + { + id: EXPERIMENT1_ID, + xpiURL: gDataRoot + EXPERIMENT1_XPI_NAME, + xpiHash: EXPERIMENT1_XPI_SHA1, + maxActiveSeconds: 10 * SEC_IN_ONE_DAY, + appName: ["XPCShell"], + channel: ["nightly"], + }, + { + id: EXPERIMENT2_ID, + xpiURL: gDataRoot + EXPERIMENT2_XPI_NAME, + xpiHash: EXPERIMENT2_XPI_SHA1, + maxActiveSeconds: 10 * SEC_IN_ONE_DAY, + appName: ["XPCShell"], + channel: ["nightly"], + }, + { + id: EXPERIMENT3_ID, + xpiURL: "https://inval.id/foo.xpi", + xpiHash: "sha1:0000000000000000000000000000000000000000", + maxActiveSeconds: 10 * SEC_IN_ONE_DAY, + appName: ["XPCShell"], + channel: ["nightly"], + }, + ], + }; + + // Setup dates for the experiments. + + let baseDate = new Date(2014, 5, 1, 12); + let startDates = []; + let endDates = []; + + for (let i = 0; i < gManifestObject.experiments.length; ++i) { + let experiment = gManifestObject.experiments[i]; + startDates.push(futureDate(baseDate, (50 + (150 * i)) * MS_IN_ONE_DAY)); + endDates .push(futureDate(startDates[i], 50 * MS_IN_ONE_DAY)); + experiment.startTime = dateToSeconds(startDates[i]); + experiment.endTime = dateToSeconds(endDates[i]); + } + + // Data to compare the result of Experiments.getExperiments() against. + + let experimentListData = [ + { + id: EXPERIMENT2_ID, + name: "Test experiment 2", + description: "And yet another experiment that experiments experimentally.", + }, + { + id: EXPERIMENT1_ID, + name: EXPERIMENT1_NAME, + description: "Yet another experiment that experiments experimentally.", + }, + ]; + + // Trigger update & re-init, clock set to before any activation. + + let now = baseDate; + defineNow(gPolicy, now); + + let experiments = new Experiments.Experiments(gPolicy); + yield experiments.updateManifest(); + let list = yield experiments.getExperiments(); + Assert.equal(list.length, 0, "Experiment list should be empty."); + checkExperimentSerializations(experiments._experiments.values()); + + yield promiseRestartManager(); + experiments = new Experiments.Experiments(gPolicy); + + yield experiments._run(); + list = yield experiments.getExperiments(); + Assert.equal(list.length, 0, "Experiment list should be empty."); + checkExperimentSerializations(experiments._experiments.values()); + + // Re-init, clock set for experiment 1 to start. + + now = futureDate(startDates[0], 5 * MS_IN_ONE_DAY); + defineNow(gPolicy, now); + + yield promiseRestartManager(); + experiments = new Experiments.Experiments(gPolicy); + yield experiments._run(); + + list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "Experiment list should have 1 entry now."); + + experimentListData[1].active = true; + experimentListData[1].endDate = now.getTime() + 10 * MS_IN_ONE_DAY; + checkExperimentListsEqual(experimentListData.slice(1), list); + checkExperimentSerializations(experiments._experiments.values()); + + let branch = yield experiments.getExperimentBranch(EXPERIMENT1_ID); + Assert.strictEqual(branch, null); + + yield experiments.setExperimentBranch(EXPERIMENT1_ID, "testbranch"); + branch = yield experiments.getExperimentBranch(EXPERIMENT1_ID); + Assert.strictEqual(branch, "testbranch"); + + // Re-init, clock set for experiment 1 to stop. + + now = futureDate(now, 20 * MS_IN_ONE_DAY); + defineNow(gPolicy, now); + + yield promiseRestartManager(); + experiments = new Experiments.Experiments(gPolicy); + yield experiments._run(); + + list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "Experiment list should have 1 entry."); + + experimentListData[1].active = false; + experimentListData[1].endDate = now.getTime(); + checkExperimentListsEqual(experimentListData.slice(1), list); + checkExperimentSerializations(experiments._experiments.values()); + + branch = yield experiments.getExperimentBranch(EXPERIMENT1_ID); + Assert.strictEqual(branch, "testbranch"); + + // Re-init, clock set for experiment 2 to start. + + now = futureDate(startDates[1], 20 * MS_IN_ONE_DAY); + defineNow(gPolicy, now); + + yield promiseRestartManager(); + experiments = new Experiments.Experiments(gPolicy); + yield experiments._run(); + + list = yield experiments.getExperiments(); + Assert.equal(list.length, 2, "Experiment list should have 2 entries."); + + experimentListData[0].active = true; + experimentListData[0].endDate = now.getTime() + 10 * MS_IN_ONE_DAY; + checkExperimentListsEqual(experimentListData, list); + checkExperimentSerializations(experiments._experiments.values()); + + // Re-init, clock set for experiment 2 to stop. + + now = futureDate(now, 20 * MS_IN_ONE_DAY); + defineNow(gPolicy, now); + + yield promiseRestartManager(); + experiments = new Experiments.Experiments(gPolicy); + yield experiments._run(); + + list = yield experiments.getExperiments(); + Assert.equal(list.length, 2, "Experiment list should have 2 entries."); + + experimentListData[0].active = false; + experimentListData[0].endDate = now.getTime(); + checkExperimentListsEqual(experimentListData, list); + checkExperimentSerializations(experiments._experiments.values()); + + // Cleanup. + + yield experiments._toggleExperimentsEnabled(false); + yield promiseRestartManager(); + yield removeCacheFile(); +}); + +add_task(function* test_expiration() { + // The manifest data we test with. + gManifestObject = { + "version": 1, + experiments: [ + { + id: EXPERIMENT1_ID, + xpiURL: gDataRoot + EXPERIMENT1_XPI_NAME, + xpiHash: EXPERIMENT1_XPI_SHA1, + maxActiveSeconds: 10 * SEC_IN_ONE_DAY, + appName: ["XPCShell"], + channel: ["nightly"], + }, + { + id: EXPERIMENT2_ID, + xpiURL: gDataRoot + EXPERIMENT2_XPI_NAME, + xpiHash: EXPERIMENT2_XPI_SHA1, + maxActiveSeconds: 50 * SEC_IN_ONE_DAY, + appName: ["XPCShell"], + channel: ["nightly"], + }, + // The 3rd experiment will never run, so it's ok to use experiment's 2 data. + { + id: EXPERIMENT3_ID, + xpiURL: gDataRoot + EXPERIMENT2_XPI_NAME, + xpiHash: EXPERIMENT2_XPI_SHA1, + maxActiveSeconds: 10 * SEC_IN_ONE_DAY, + appName: ["XPCShell"], + channel: ["nightly"], + } + ], + }; + + // Data to compare the result of Experiments.getExperiments() against. + let experimentListData = [ + { + id: EXPERIMENT2_ID, + name: "Test experiment 2", + description: "And yet another experiment that experiments experimentally.", + }, + { + id: EXPERIMENT1_ID, + name: EXPERIMENT1_NAME, + description: "Yet another experiment that experiments experimentally.", + }, + ]; + + // Setup dates for the experiments. + let baseDate = new Date(2014, 5, 1, 12); + let startDates = []; + let endDates = []; + + for (let i = 0; i < gManifestObject.experiments.length; ++i) { + let experiment = gManifestObject.experiments[i]; + // Spread out experiments in time so that one experiment can end and expire while + // the next is still running. + startDates.push(futureDate(baseDate, (50 + (200 * i)) * MS_IN_ONE_DAY)); + endDates .push(futureDate(startDates[i], 50 * MS_IN_ONE_DAY)); + experiment.startTime = dateToSeconds(startDates[i]); + experiment.endTime = dateToSeconds(endDates[i]); + } + + let now = null; + let experiments = null; + + let setDateAndRestartExperiments = new Task.async(function* (newDate) { + now = newDate; + defineNow(gPolicy, now); + + yield promiseRestartManager(); + experiments = new Experiments.Experiments(gPolicy); + yield experiments._run(); + }); + + // Trigger update & re-init, clock set to before any activation. + now = baseDate; + defineNow(gPolicy, now); + + experiments = new Experiments.Experiments(gPolicy); + yield experiments.updateManifest(); + let list = yield experiments.getExperiments(); + Assert.equal(list.length, 0, "Experiment list should be empty."); + + // Re-init, clock set for experiment 1 to start... + yield setDateAndRestartExperiments(startDates[0]); + list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "The first experiment should have started."); + + // ... init again, and set the clock so that the first experiment ends. + yield setDateAndRestartExperiments(endDates[0]); + + // The experiment just ended, it should still be in the cache, but marked + // as finished. + list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "Experiment list should have 1 entry."); + + experimentListData[1].active = false; + experimentListData[1].endDate = now.getTime(); + checkExperimentListsEqual(experimentListData.slice(1), list); + validateCache([...experiments._experiments.keys()], [EXPERIMENT1_ID, EXPERIMENT2_ID, EXPERIMENT3_ID]); + + // Start the second experiment. + yield setDateAndRestartExperiments(startDates[1]); + + // The experiments cache should contain the finished experiment and the + // one that's still running. + list = yield experiments.getExperiments(); + Assert.equal(list.length, 2, "Experiment list should have 2 entries."); + + experimentListData[0].active = true; + experimentListData[0].endDate = now.getTime() + 50 * MS_IN_ONE_DAY; + checkExperimentListsEqual(experimentListData, list); + + // Move the clock in the future, just 31 days after the start date of the second experiment, + // so that the cache for the first experiment expires and the second experiment is still running. + yield setDateAndRestartExperiments(futureDate(startDates[1], 31 * MS_IN_ONE_DAY)); + validateCache([...experiments._experiments.keys()], [EXPERIMENT2_ID, EXPERIMENT3_ID]); + + // Make sure that the expired experiment is not reported anymore. + let history = yield experiments.getExperiments(); + Assert.equal(history.length, 1, "Experiments older than 180 days must be removed from the cache."); + + // Test that we don't write expired experiments in the cache. + yield setDateAndRestartExperiments(now); + validateCache([...experiments._experiments.keys()], [EXPERIMENT2_ID, EXPERIMENT3_ID]); + + // The first experiment should be expired and not in the cache, it ended more than + // 180 days ago. We should see the one still running in the cache. + history = yield experiments.getExperiments(); + Assert.equal(history.length, 1, "Expired experiments must not be saved to cache."); + checkExperimentListsEqual(experimentListData.slice(0, 1), history); + + // Test that experiments that are cached locally but never ran are removed from cache + // when they are removed from the manifest (this is cached data, not really history). + gManifestObject["experiments"] = gManifestObject["experiments"].slice(1, 1); + yield experiments.updateManifest(); + validateCache([...experiments._experiments.keys()], [EXPERIMENT2_ID]); + + // Cleanup. + yield experiments._toggleExperimentsEnabled(false); + yield promiseRestartManager(); + yield removeCacheFile(); +}); diff --git a/application/basilisk/experiments/test/xpcshell/test_cacherace.js b/application/basilisk/experiments/test/xpcshell/test_cacherace.js new file mode 100644 index 000000000..e1b788b2e --- /dev/null +++ b/application/basilisk/experiments/test/xpcshell/test_cacherace.js @@ -0,0 +1,102 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +Cu.import("resource://testing-common/httpd.js"); +Cu.import("resource://gre/modules/Timer.jsm"); + +const MANIFEST_HANDLER = "manifests/handler"; + +const SEC_IN_ONE_DAY = 24 * 60 * 60; +const MS_IN_ONE_DAY = SEC_IN_ONE_DAY * 1000; + +var gHttpServer = null; +var gHttpRoot = null; +var gDataRoot = null; +var gPolicy = null; +var gManifestObject = null; +var gManifestHandlerURI = null; + +function run_test() { + run_next_test(); +} + +add_task(function* test_setup() { + loadAddonManager(); + yield removeCacheFile(); + + gHttpServer = new HttpServer(); + gHttpServer.start(-1); + let port = gHttpServer.identity.primaryPort; + gHttpRoot = "http://localhost:" + port + "/"; + gDataRoot = gHttpRoot + "data/"; + gManifestHandlerURI = gHttpRoot + MANIFEST_HANDLER; + gHttpServer.registerDirectory("/data/", do_get_cwd()); + gHttpServer.registerPathHandler("/" + MANIFEST_HANDLER, (request, response) => { + response.setStatusLine(null, 200, "OK"); + response.write(JSON.stringify(gManifestObject)); + response.processAsync(); + response.finish(); + }); + do_register_cleanup(() => gHttpServer.stop(() => {})); + + Services.prefs.setBoolPref(PREF_EXPERIMENTS_ENABLED, true); + Services.prefs.setIntPref(PREF_LOGGING_LEVEL, 0); + Services.prefs.setBoolPref(PREF_LOGGING_DUMP, true); + Services.prefs.setCharPref(PREF_MANIFEST_URI, gManifestHandlerURI); + Services.prefs.setIntPref(PREF_FETCHINTERVAL, 0); + + let ExperimentsScope = Cu.import("resource:///modules/experiments/Experiments.jsm", {}); + let Experiments = ExperimentsScope.Experiments; + + gPolicy = new Experiments.Policy(); + patchPolicy(gPolicy, { + updatechannel: () => "nightly", + delayCacheWrite: (promise) => { + return new Promise((resolve, reject) => { + promise.then( + (result) => { setTimeout(() => resolve(result), 500); }, + (err) => { reject(err); } + ); + }); + }, + }); + + let now = new Date(2014, 5, 1, 12); + defineNow(gPolicy, now); + + let experimentName = "experiment-racybranch.xpi"; + let experimentPath = getExperimentPath(experimentName); + let experimentHash = "sha1:" + sha1File(experimentPath); + + gManifestObject = { + version: 1, + experiments: [ + { + id: "test-experiment-racybranch@tests.mozilla.org", + xpiURL: gDataRoot + "experiment-racybranch.xpi", + xpiHash: experimentHash, + maxActiveSeconds: 10 * SEC_IN_ONE_DAY, + appName: ["XPCShell"], + channel: ["nightly"], + startTime: dateToSeconds(futureDate(now, -MS_IN_ONE_DAY)), + endTime: dateToSeconds(futureDate(now, MS_IN_ONE_DAY)), + }, + ], + }; + + do_print("gManifestObject: " + JSON.stringify(gManifestObject)); + + // In order for the addon manager to work properly, we hack + // Experiments.instance which is used by the XPIProvider + let experiments = new Experiments.Experiments(gPolicy); + Assert.strictEqual(ExperimentsScope.gExperiments, null); + ExperimentsScope.gExperiments = experiments; + + yield experiments.updateManifest(); + let active = experiments._getActiveExperiment(); + Assert.ok(active); + Assert.equal(active.branch, "racy-set"); + Assert.ok(!experiments._dirty); +}); diff --git a/application/basilisk/experiments/test/xpcshell/test_conditions.js b/application/basilisk/experiments/test/xpcshell/test_conditions.js new file mode 100644 index 000000000..aa0784a0e --- /dev/null +++ b/application/basilisk/experiments/test/xpcshell/test_conditions.js @@ -0,0 +1,325 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + + +Cu.import("resource:///modules/experiments/Experiments.jsm"); +Cu.import("resource://gre/modules/TelemetryController.jsm", this); + +const SEC_IN_ONE_DAY = 24 * 60 * 60; + +var gPolicy = null; + +function ManifestEntry(data) { + this.id = EXPERIMENT1_ID; + this.xpiURL = "http://localhost:1/dummy.xpi"; + this.xpiHash = EXPERIMENT1_XPI_SHA1; + this.startTime = new Date(2010, 0, 1, 12).getTime() / 1000; + this.endTime = new Date(9001, 0, 1, 12).getTime() / 1000; + this.maxActiveSeconds = SEC_IN_ONE_DAY; + this.appName = ["XPCShell"]; + this.channel = ["nightly"]; + + data = data || {}; + for (let k of Object.keys(data)) { + this[k] = data[k]; + } + + if (!this.endTime) { + this.endTime = this.startTime + 5 * SEC_IN_ONE_DAY; + } +} + +function applicableFromManifestData(data, policy) { + let manifestData = new ManifestEntry(data); + let entry = new Experiments.ExperimentEntry(policy); + entry.initFromManifestData(manifestData); + return entry.isApplicable(); +} + +function run_test() { + run_next_test(); +} + +add_task(function* test_setup() { + createAppInfo(); + do_get_profile(); + startAddonManagerOnly(); + yield TelemetryController.testSetup(); + gPolicy = new Experiments.Policy(); + + patchPolicy(gPolicy, { + updatechannel: () => "nightly", + locale: () => "en-US", + random: () => 0.5, + }); + + Services.prefs.setBoolPref(PREF_EXPERIMENTS_ENABLED, true); + Services.prefs.setIntPref(PREF_LOGGING_LEVEL, 0); + Services.prefs.setBoolPref(PREF_LOGGING_DUMP, true); +}); + +function arraysEqual(a, b) { + if (a.length !== b.length) { + return false; + } + + for (let i = 0; i < a.length; ++i) { + if (a[i] !== b[i]) { + return false; + } + } + + return true; +} + +// This function exists solely to be .toSource()d +const sanityFilter = function filter(c) { + if (c.telemetryEnvironment === undefined) { + throw Error("No .telemetryEnvironment"); + } + if (c.telemetryEnvironment.build == undefined) { + throw Error("No .telemetryEnvironment.build"); + } + return true; +} + +// Utility function to generate build ID for previous/next date. +function addDate(buildId, diff) { + let m = /^([0-9]{4})([0-9]{2})([0-9]{2})(.*)$/.exec(buildId); + if (!m) { + throw Error("Unsupported build ID: " + buildId); + } + let year = Number.parseInt(m[1], 10); + let month = Number.parseInt(m[2], 10); + let date = Number.parseInt(m[3], 10); + let remainingParts = m[4]; + + let d = new Date(); + d.setUTCFullYear(year, month - 1, date); + d.setTime(d.getTime() + diff * 24 * 60 * 60 * 1000); + + let yearStr = String(d.getUTCFullYear()); + let monthStr = ("0" + String(d.getUTCMonth() + 1)).slice(-2); + let dateStr = ("0" + String(d.getUTCDate())).slice(-2); + return yearStr + monthStr + dateStr + remainingParts; +} +function prevDate(buildId) { + return addDate(buildId, -1); +} +function nextDate(buildId) { + return addDate(buildId, 1); +} + +add_task(function* test_simpleFields() { + let testData = [ + // "expected applicable?", failure reason or null, manifest data + + // misc. environment + + [false, ["appName"], {appName: []}], + [false, ["appName"], {appName: ["foo", gAppInfo.name + "-invalid"]}], + [true, null, {appName: ["not-an-app-name", gAppInfo.name]}], + + [false, ["os"], {os: []}], + [false, ["os"], {os: ["42", "abcdef"]}], + [true, null, {os: [gAppInfo.OS, "plan9"]}], + + [false, ["channel"], {channel: []}], + [false, ["channel"], {channel: ["foo", gPolicy.updatechannel() + "-invalid"]}], + [true, null, {channel: ["not-a-channel", gPolicy.updatechannel()]}], + + [false, ["locale"], {locale: []}], + [false, ["locale"], {locale: ["foo", gPolicy.locale + "-invalid"]}], + [true, null, {locale: ["not-a-locale", gPolicy.locale()]}], + + // version + + [false, ["version"], {version: []}], + [false, ["version"], {version: ["-1", gAppInfo.version + "-invalid", "asdf", "0,4", "99.99", "0.1.1.1"]}], + [true, null, {version: ["99999999.999", "-1", gAppInfo.version]}], + + [false, ["minVersion"], {minVersion: "1.0.1"}], + [true, null, {minVersion: "1.0b1"}], + [true, null, {minVersion: "1.0"}], + [true, null, {minVersion: "0.9"}], + + [false, ["maxVersion"], {maxVersion: "0.1"}], + [false, ["maxVersion"], {maxVersion: "0.9.9"}], + [false, ["maxVersion"], {maxVersion: "1.0b1"}], + [true, ["maxVersion"], {maxVersion: "1.0"}], + [true, ["maxVersion"], {maxVersion: "1.7pre"}], + + // build id + + [false, ["buildIDs"], {buildIDs: []}], + [false, ["buildIDs"], {buildIDs: ["not-a-build-id", gAppInfo.platformBuildID + "-invalid"]}], + [true, null, {buildIDs: ["not-a-build-id", gAppInfo.platformBuildID]}], + + [true, null, {minBuildID: prevDate(gAppInfo.platformBuildID)}], + [true, null, {minBuildID: gAppInfo.platformBuildID}], + [false, ["minBuildID"], {minBuildID: nextDate(gAppInfo.platformBuildID)}], + + [false, ["maxBuildID"], {maxBuildID: prevDate(gAppInfo.platformBuildID)}], + [true, null, {maxBuildID: gAppInfo.platformBuildID}], + [true, null, {maxBuildID: nextDate(gAppInfo.platformBuildID)}], + + // sample + + [false, ["sample"], {sample: -1 }], + [false, ["sample"], {sample: 0.0}], + [false, ["sample"], {sample: 0.1}], + [true, null, {sample: 0.5}], + [true, null, {sample: 0.6}], + [true, null, {sample: 1.0}], + [true, null, {sample: 0.5}], + + // experiment control + + [false, ["disabled"], {disabled: true}], + [true, null, {disabled: false}], + + [false, ["frozen"], {frozen: true}], + [true, null, {frozen: false}], + + [false, null, {frozen: true, disabled: true}], + [false, null, {frozen: true, disabled: false}], + [false, null, {frozen: false, disabled: true}], + [true, null, {frozen: false, disabled: false}], + + // jsfilter + + [true, null, {jsfilter: "function filter(c) { return true; }"}], + [false, ["jsfilter-false"], {jsfilter: "function filter(c) { return false; }"}], + [true, null, {jsfilter: "function filter(c) { return 123; }"}], // truthy + [false, ["jsfilter-false"], {jsfilter: "function filter(c) { return ''; }"}], // falsy + [false, ["jsfilter-false"], {jsfilter: "function filter(c) { var a = []; }"}], // undefined + [false, ["jsfilter-threw", "some error"], {jsfilter: "function filter(c) { throw new Error('some error'); }"}], + [false, ["jsfilter-evalfailed"], {jsfilter: "123, this won't work"}], + [true, null, {jsfilter: "var filter = " + sanityFilter.toSource()}], + ]; + + for (let i = 0; i < testData.length; ++i) { + let entry = testData[i]; + let applicable; + let reason = null; + + yield applicableFromManifestData(entry[2], gPolicy).then( + value => applicable = value, + value => { + applicable = false; + reason = value; + } + ); + + Assert.equal(applicable, entry[0], + "Experiment entry applicability should match for test " + + i + ": " + JSON.stringify(entry[2])); + + let expectedReason = entry[1]; + if (!applicable && expectedReason) { + Assert.ok(arraysEqual(reason, expectedReason), + "Experiment rejection reasons should match for test " + i + ". " + + "Got " + JSON.stringify(reason) + ", expected " + + JSON.stringify(expectedReason)); + } + } +}); + +add_task(function* test_times() { + let now = new Date(2014, 5, 6, 12); + let nowSec = now.getTime() / 1000; + let testData = [ + // "expected applicable?", rejection reason or null, fake now date, manifest data + + // start time + + [true, null, now, + {startTime: nowSec - 5 * SEC_IN_ONE_DAY, + endTime: nowSec + 10 * SEC_IN_ONE_DAY}], + [true, null, now, + {startTime: nowSec, + endTime: nowSec + 10 * SEC_IN_ONE_DAY}], + [false, "startTime", now, + {startTime: nowSec + 5 * SEC_IN_ONE_DAY, + endTime: nowSec + 10 * SEC_IN_ONE_DAY}], + + // end time + + [false, "endTime", now, + {startTime: nowSec - 5 * SEC_IN_ONE_DAY, + endTime: nowSec - 10 * SEC_IN_ONE_DAY}], + [false, "endTime", now, + {startTime: nowSec - 5 * SEC_IN_ONE_DAY, + endTime: nowSec - 5 * SEC_IN_ONE_DAY}], + + // max start time + + [false, "maxStartTime", now, + {maxStartTime: nowSec - 15 * SEC_IN_ONE_DAY, + startTime: nowSec - 10 * SEC_IN_ONE_DAY, + endTime: nowSec + 10 * SEC_IN_ONE_DAY}], + [false, "maxStartTime", now, + {maxStartTime: nowSec - 1 * SEC_IN_ONE_DAY, + startTime: nowSec - 10 * SEC_IN_ONE_DAY, + endTime: nowSec + 10 * SEC_IN_ONE_DAY}], + [false, "maxStartTime", now, + {maxStartTime: nowSec - 10 * SEC_IN_ONE_DAY, + startTime: nowSec - 10 * SEC_IN_ONE_DAY, + endTime: nowSec + 10 * SEC_IN_ONE_DAY}], + [true, null, now, + {maxStartTime: nowSec, + startTime: nowSec - 10 * SEC_IN_ONE_DAY, + endTime: nowSec + 10 * SEC_IN_ONE_DAY}], + [true, null, now, + {maxStartTime: nowSec + 1 * SEC_IN_ONE_DAY, + startTime: nowSec - 10 * SEC_IN_ONE_DAY, + endTime: nowSec + 10 * SEC_IN_ONE_DAY}], + + // max active seconds + + [true, null, now, + {maxActiveSeconds: 5 * SEC_IN_ONE_DAY, + startTime: nowSec - 10 * SEC_IN_ONE_DAY, + endTime: nowSec + 10 * SEC_IN_ONE_DAY}], + [true, null, now, + {maxActiveSeconds: 10 * SEC_IN_ONE_DAY, + startTime: nowSec - 10 * SEC_IN_ONE_DAY, + endTime: nowSec + 10 * SEC_IN_ONE_DAY}], + [true, null, now, + {maxActiveSeconds: 15 * SEC_IN_ONE_DAY, + startTime: nowSec - 10 * SEC_IN_ONE_DAY, + endTime: nowSec + 10 * SEC_IN_ONE_DAY}], + [true, null, now, + {maxActiveSeconds: 20 * SEC_IN_ONE_DAY, + startTime: nowSec - 10 * SEC_IN_ONE_DAY, + endTime: nowSec + 10 * SEC_IN_ONE_DAY}], + ]; + + for (let i = 0; i < testData.length; ++i) { + let entry = testData[i]; + let applicable; + let reason = null; + defineNow(gPolicy, entry[2]); + + yield applicableFromManifestData(entry[3], gPolicy).then( + value => applicable = value, + value => { + applicable = false; + reason = value; + } + ); + + Assert.equal(applicable, entry[0], + "Experiment entry applicability should match for test " + + i + ": " + JSON.stringify([entry[2], entry[3]])); + if (!applicable && entry[1]) { + Assert.equal(reason, entry[1], "Experiment rejection reason should match for test " + i); + } + } +}); + +add_task(function* test_shutdown() { + yield TelemetryController.testShutdown(); +}); diff --git a/application/basilisk/experiments/test/xpcshell/test_disableExperiments.js b/application/basilisk/experiments/test/xpcshell/test_disableExperiments.js new file mode 100644 index 000000000..9b3fef993 --- /dev/null +++ b/application/basilisk/experiments/test/xpcshell/test_disableExperiments.js @@ -0,0 +1,180 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +Cu.import("resource://testing-common/httpd.js"); +Cu.import("resource://testing-common/AddonManagerTesting.jsm"); + +XPCOMUtils.defineLazyModuleGetter(this, "Experiments", + "resource:///modules/experiments/Experiments.jsm"); + +const MANIFEST_HANDLER = "manifests/handler"; + +const SEC_IN_ONE_DAY = 24 * 60 * 60; +const MS_IN_ONE_DAY = SEC_IN_ONE_DAY * 1000; + +var gHttpServer = null; +var gHttpRoot = null; +var gDataRoot = null; +var gPolicy = null; +var gManifestObject = null; +var gManifestHandlerURI = null; + +function run_test() { + run_next_test(); +} + +add_task(function* test_setup() { + loadAddonManager(); + + gHttpServer = new HttpServer(); + gHttpServer.start(-1); + let port = gHttpServer.identity.primaryPort; + gHttpRoot = "http://localhost:" + port + "/"; + gDataRoot = gHttpRoot + "data/"; + gManifestHandlerURI = gHttpRoot + MANIFEST_HANDLER; + gHttpServer.registerDirectory("/data/", do_get_cwd()); + gHttpServer.registerPathHandler("/" + MANIFEST_HANDLER, (request, response) => { + response.setStatusLine(null, 200, "OK"); + response.write(JSON.stringify(gManifestObject)); + response.processAsync(); + response.finish(); + }); + do_register_cleanup(() => gHttpServer.stop(() => {})); + + Services.prefs.setBoolPref(PREF_EXPERIMENTS_ENABLED, true); + Services.prefs.setIntPref(PREF_LOGGING_LEVEL, 0); + Services.prefs.setBoolPref(PREF_LOGGING_DUMP, true); + Services.prefs.setCharPref(PREF_MANIFEST_URI, gManifestHandlerURI); + Services.prefs.setIntPref(PREF_FETCHINTERVAL, 0); + + gPolicy = new Experiments.Policy(); + patchPolicy(gPolicy, { + updatechannel: () => "nightly", + oneshotTimer: (callback, timeout, thisObj, name) => {}, + }); +}); + +// Test disabling the feature stops current and future experiments. + +add_task(function* test_disableExperiments() { + const OBSERVER_TOPIC = "experiments-changed"; + let observerFireCount = 0; + let expectedObserverFireCount = 0; + let observer = () => ++observerFireCount; + Services.obs.addObserver(observer, OBSERVER_TOPIC, false); + + // Dates the following tests are based on. + + let baseDate = new Date(2014, 5, 1, 12); + let startDate1 = futureDate(baseDate, 50 * MS_IN_ONE_DAY); + let endDate1 = futureDate(baseDate, 100 * MS_IN_ONE_DAY); + let startDate2 = futureDate(baseDate, 150 * MS_IN_ONE_DAY); + let endDate2 = futureDate(baseDate, 200 * MS_IN_ONE_DAY); + + // The manifest data we test with. + + gManifestObject = { + "version": 1, + experiments: [ + { + id: EXPERIMENT2_ID, + xpiURL: gDataRoot + EXPERIMENT2_XPI_NAME, + xpiHash: EXPERIMENT2_XPI_SHA1, + startTime: dateToSeconds(startDate2), + endTime: dateToSeconds(endDate2), + maxActiveSeconds: 10 * SEC_IN_ONE_DAY, + appName: ["XPCShell"], + channel: ["nightly"], + }, + { + id: EXPERIMENT1_ID, + xpiURL: gDataRoot + EXPERIMENT1_XPI_NAME, + xpiHash: EXPERIMENT1_XPI_SHA1, + startTime: dateToSeconds(startDate1), + endTime: dateToSeconds(endDate1), + maxActiveSeconds: 10 * SEC_IN_ONE_DAY, + appName: ["XPCShell"], + channel: ["nightly"], + }, + ], + }; + + let experiments = new Experiments.Experiments(gPolicy); + + // Trigger update, clock set to before any activation. + // Use updateManifest() to provide for coverage of that path. + + let now = baseDate; + defineNow(gPolicy, now); + + yield experiments.updateManifest(); + Assert.equal(observerFireCount, ++expectedObserverFireCount, + "Experiments observer should have been called."); + let list = yield experiments.getExperiments(); + Assert.equal(list.length, 0, "Experiment list should be empty."); + let addons = yield getExperimentAddons(); + Assert.equal(addons.length, 0, "Precondition: No experiment add-ons are installed."); + + // Trigger update, clock set for experiment 1 to start. + + now = futureDate(startDate1, 5 * MS_IN_ONE_DAY); + defineNow(gPolicy, now); + + yield experiments.updateManifest(); + Assert.equal(observerFireCount, ++expectedObserverFireCount, + "Experiments observer should have been called."); + + list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "Experiment list should have 1 entry now."); + Assert.equal(list[0].active, true, "Experiment should be active."); + addons = yield getExperimentAddons(); + Assert.equal(addons.length, 1, "An experiment add-on was installed."); + + // Disable the experiments feature. Check that we stop the running experiment. + + Services.prefs.setBoolPref(PREF_EXPERIMENTS_ENABLED, false); + yield experiments._mainTask; + + Assert.equal(observerFireCount, ++expectedObserverFireCount, + "Experiments observer should have been called."); + + list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "Experiment list should have 1 entry."); + Assert.equal(list[0].active, false, "Experiment entry should not be active."); + addons = yield getExperimentAddons(); + Assert.equal(addons.length, 0, "The experiment add-on should be uninstalled."); + + // Trigger update, clock set for experiment 2 to start. Verify we don't start it. + + now = startDate2; + defineNow(gPolicy, now); + + try { + yield experiments.updateManifest(); + } catch (e) { + // This exception is expected, we rethrow everything else + if (e.message != "experiments are disabled") { + throw e; + } + } + + experiments.notify(); + yield experiments._mainTask; + + Assert.equal(observerFireCount, expectedObserverFireCount, + "Experiments observer should not have been called."); + + list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "Experiment list should still have 1 entry."); + Assert.equal(list[0].active, false, "Experiment entry should not be active."); + addons = yield getExperimentAddons(); + Assert.equal(addons.length, 0, "There should still be no experiment add-on installed."); + + // Cleanup. + + Services.obs.removeObserver(observer, OBSERVER_TOPIC); + yield promiseRestartManager(); + yield removeCacheFile(); +}); diff --git a/application/basilisk/experiments/test/xpcshell/test_fetch.js b/application/basilisk/experiments/test/xpcshell/test_fetch.js new file mode 100644 index 000000000..e8d76fa35 --- /dev/null +++ b/application/basilisk/experiments/test/xpcshell/test_fetch.js @@ -0,0 +1,68 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +Cu.import("resource://testing-common/httpd.js"); +Cu.import("resource://gre/modules/Services.jsm"); +Cu.import("resource://gre/modules/osfile.jsm"); +Cu.import("resource:///modules/experiments/Experiments.jsm"); + +var gHttpServer = null; +var gHttpRoot = null; +var gPolicy = new Experiments.Policy(); + +function run_test() { + loadAddonManager(); + + gHttpServer = new HttpServer(); + gHttpServer.start(-1); + let port = gHttpServer.identity.primaryPort; + gHttpRoot = "http://localhost:" + port + "/"; + gHttpServer.registerDirectory("/", do_get_cwd()); + do_register_cleanup(() => gHttpServer.stop(() => {})); + + Services.prefs.setBoolPref(PREF_EXPERIMENTS_ENABLED, true); + Services.prefs.setIntPref(PREF_LOGGING_LEVEL, 0); + Services.prefs.setBoolPref(PREF_LOGGING_DUMP, true); + + patchPolicy(gPolicy, { + updatechannel: () => "nightly", + }); + + run_next_test(); +} + +add_task(function* test_fetchAndCache() { + Services.prefs.setCharPref(PREF_MANIFEST_URI, gHttpRoot + "experiments_1.manifest"); + let ex = new Experiments.Experiments(gPolicy); + + Assert.equal(ex._experiments, null, "There should be no cached experiments yet."); + yield ex.updateManifest(); + Assert.notEqual(ex._experiments.size, 0, "There should be cached experiments now."); + + yield promiseRestartManager(); +}); + +add_task(function* test_checkCache() { + let ex = new Experiments.Experiments(gPolicy); + yield ex.notify(); + Assert.notEqual(ex._experiments.size, 0, "There should be cached experiments now."); + + yield promiseRestartManager(); +}); + +add_task(function* test_fetchInvalid() { + yield removeCacheFile(); + + Services.prefs.setCharPref(PREF_MANIFEST_URI, gHttpRoot + "experiments_1.manifest"); + let ex = new Experiments.Experiments(gPolicy); + yield ex.updateManifest(); + Assert.notEqual(ex._experiments.size, 0, "There should be experiments"); + + Services.prefs.setCharPref(PREF_MANIFEST_URI, gHttpRoot + "invalid.manifest"); + yield ex.updateManifest() + Assert.notEqual(ex._experiments.size, 0, "There should still be experiments: fetch failure shouldn't remove them."); + + yield promiseRestartManager(); +}); diff --git a/application/basilisk/experiments/test/xpcshell/test_nethang_bug1012924.js b/application/basilisk/experiments/test/xpcshell/test_nethang_bug1012924.js new file mode 100644 index 000000000..7ef604901 --- /dev/null +++ b/application/basilisk/experiments/test/xpcshell/test_nethang_bug1012924.js @@ -0,0 +1,47 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +Cu.import("resource://testing-common/httpd.js"); +Cu.import("resource:///modules/experiments/Experiments.jsm"); + +const MANIFEST_HANDLER = "manifests/handler"; + +function run_test() { + run_next_test(); +} + +add_task(function* test_setup() { + loadAddonManager(); + do_get_profile(); + + let httpServer = new HttpServer(); + httpServer.start(-1); + let port = httpServer.identity.primaryPort; + let httpRoot = "http://localhost:" + port + "/"; + let handlerURI = httpRoot + MANIFEST_HANDLER; + httpServer.registerPathHandler("/" + MANIFEST_HANDLER, + (request, response) => { + response.processAsync(); + response.setStatus(null, 200, "OK"); + response.write("["); // never finish! + }); + + do_register_cleanup(() => httpServer.stop(() => {})); + Services.prefs.setBoolPref(PREF_EXPERIMENTS_ENABLED, true); + Services.prefs.setIntPref(PREF_LOGGING_LEVEL, 0); + Services.prefs.setBoolPref(PREF_LOGGING_DUMP, true); + Services.prefs.setCharPref(PREF_MANIFEST_URI, handlerURI); + Services.prefs.setIntPref(PREF_FETCHINTERVAL, 0); + + let experiments = Experiments.instance(); + experiments.updateManifest().then( + () => { + Assert.ok(true, "updateManifest finished successfully"); + }, + (e) => { + do_throw("updateManifest should not have failed: got error " + e); + }); + yield experiments.uninit(); +}); diff --git a/application/basilisk/experiments/test/xpcshell/test_previous_provider.js b/application/basilisk/experiments/test/xpcshell/test_previous_provider.js new file mode 100644 index 000000000..ddd1cd588 --- /dev/null +++ b/application/basilisk/experiments/test/xpcshell/test_previous_provider.js @@ -0,0 +1,177 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +Cu.import("resource://gre/modules/Promise.jsm"); +Cu.import("resource:///modules/experiments/Experiments.jsm"); +Cu.import("resource://testing-common/httpd.js"); + +var gDataRoot; +var gHttpServer; +var gManifestObject; + +function run_test() { + run_next_test(); +} + +add_task(function test_setup() { + loadAddonManager(); + do_get_profile(); + + gHttpServer = new HttpServer(); + gHttpServer.start(-1); + let httpRoot = "http://localhost:" + gHttpServer.identity.primaryPort + "/"; + gDataRoot = httpRoot + "data/"; + gHttpServer.registerDirectory("/data/", do_get_cwd()); + gHttpServer.registerPathHandler("/manifests/handler", (req, res) => { + res.setStatusLine(null, 200, "OK"); + res.write(JSON.stringify(gManifestObject)); + res.processAsync(); + res.finish(); + }); + do_register_cleanup(() => gHttpServer.stop(() => {})); + + Services.prefs.setBoolPref("experiments.enabled", true); + Services.prefs.setCharPref("experiments.manifest.uri", + httpRoot + "manifests/handler"); + Services.prefs.setBoolPref("experiments.logging.dump", true); + Services.prefs.setCharPref("experiments.logging.level", "Trace"); +}); + +add_task(function* test_provider_basic() { + let e = Experiments.instance(); + + let provider = new Experiments.PreviousExperimentProvider(e); + e._setPreviousExperimentsProvider(provider); + + let deferred = Promise.defer(); + provider.getAddonsByTypes(["experiment"], (addons) => { + deferred.resolve(addons); + }); + let experimentAddons = yield deferred.promise; + Assert.ok(Array.isArray(experimentAddons), "getAddonsByTypes returns an Array."); + Assert.equal(experimentAddons.length, 0, "No previous add-ons returned."); + + gManifestObject = { + version: 1, + experiments: [ + { + id: EXPERIMENT1_ID, + xpiURL: gDataRoot + EXPERIMENT1_XPI_NAME, + xpiHash: EXPERIMENT1_XPI_SHA1, + startTime: Date.now() / 1000 - 60, + endTime: Date.now() / 1000 + 60, + maxActiveSeconds: 60, + appName: ["XPCShell"], + channel: [e._policy.updatechannel()], + }, + ], + }; + + yield e.updateManifest(); + + deferred = Promise.defer(); + provider.getAddonsByTypes(["experiment"], (addons) => { + deferred.resolve(addons); + }); + experimentAddons = yield deferred.promise; + Assert.equal(experimentAddons.length, 0, "Still no previous experiment."); + + let experiments = yield e.getExperiments(); + Assert.equal(experiments.length, 1, "1 experiment present."); + Assert.ok(experiments[0].active, "It is active."); + + // Deactivate it. + defineNow(e._policy, new Date(gManifestObject.experiments[0].endTime * 1000 + 1000)); + yield e.updateManifest(); + + experiments = yield e.getExperiments(); + Assert.equal(experiments.length, 1, "1 experiment present."); + Assert.equal(experiments[0].active, false, "It isn't active."); + + deferred = Promise.defer(); + provider.getAddonsByTypes(["experiment"], (addons) => { + deferred.resolve(addons); + }); + experimentAddons = yield deferred.promise; + Assert.equal(experimentAddons.length, 1, "1 previous add-on known."); + Assert.equal(experimentAddons[0].id, EXPERIMENT1_ID, "ID matches expected."); + + deferred = Promise.defer(); + provider.getAddonByID(EXPERIMENT1_ID, (addon) => { + deferred.resolve(addon); + }); + let addon = yield deferred.promise; + Assert.ok(addon, "We got an add-on from its ID."); + Assert.equal(addon.id, EXPERIMENT1_ID, "ID matches expected."); + Assert.ok(addon.appDisabled, "Add-on is a previous experiment."); + Assert.ok(addon.userDisabled, "Add-on is disabled."); + Assert.equal(addon.type, "experiment", "Add-on is an experiment."); + Assert.equal(addon.isActive, false, "Add-on is not active."); + Assert.equal(addon.permissions, 0, "Add-on has no permissions."); + + deferred = Promise.defer(); + AddonManager.getAddonsByTypes(["experiment"], (addons) => { + deferred.resolve(addons); + }); + experimentAddons = yield deferred.promise; + Assert.equal(experimentAddons.length, 1, "Got 1 experiment from add-on manager."); + Assert.equal(experimentAddons[0].id, EXPERIMENT1_ID, "ID matches expected."); + Assert.ok(experimentAddons[0].appDisabled, "It is a previous experiment add-on."); +}); + +add_task(function* test_active_and_previous() { + // Building on the previous test, activate experiment 2. + let e = Experiments.instance(); + let provider = new Experiments.PreviousExperimentProvider(e); + e._setPreviousExperimentsProvider(provider); + + gManifestObject = { + version: 1, + experiments: [ + { + id: EXPERIMENT2_ID, + xpiURL: gDataRoot + EXPERIMENT2_XPI_NAME, + xpiHash: EXPERIMENT2_XPI_SHA1, + startTime: Date.now() / 1000 - 60, + endTime: Date.now() / 1000 + 60, + maxActiveSeconds: 60, + appName: ["XPCShell"], + channel: [e._policy.updatechannel()], + }, + ], + }; + + defineNow(e._policy, new Date()); + yield e.updateManifest(); + + let experiments = yield e.getExperiments(); + Assert.equal(experiments.length, 2, "2 experiments known."); + + let deferred = Promise.defer(); + provider.getAddonsByTypes(["experiment"], (addons) => { + deferred.resolve(addons); + }); + let experimentAddons = yield deferred.promise; + Assert.equal(experimentAddons.length, 1, "1 previous experiment."); + + deferred = Promise.defer(); + AddonManager.getAddonsByTypes(["experiment"], (addons) => { + deferred.resolve(addons); + }); + experimentAddons = yield deferred.promise; + Assert.equal(experimentAddons.length, 2, "2 experiment add-ons known."); + + for (let addon of experimentAddons) { + if (addon.id == EXPERIMENT1_ID) { + Assert.equal(addon.isActive, false, "Add-on is not active."); + Assert.ok(addon.appDisabled, "Should be a previous experiment."); + } else if (addon.id == EXPERIMENT2_ID) { + Assert.ok(addon.isActive, "Add-on is active."); + Assert.ok(!addon.appDisabled, "Should not be a previous experiment."); + } else { + throw new Error("Unexpected add-on ID: " + addon.id); + } + } +}); diff --git a/application/basilisk/experiments/test/xpcshell/test_telemetry.js b/application/basilisk/experiments/test/xpcshell/test_telemetry.js new file mode 100644 index 000000000..2dda27721 --- /dev/null +++ b/application/basilisk/experiments/test/xpcshell/test_telemetry.js @@ -0,0 +1,293 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +Cu.import("resource://testing-common/httpd.js"); +Cu.import("resource://gre/modules/TelemetryLog.jsm"); +var {TELEMETRY_LOG, Experiments} = Cu.import("resource:///modules/experiments/Experiments.jsm", {}); + + +const MANIFEST_HANDLER = "manifests/handler"; + +const SEC_IN_ONE_DAY = 24 * 60 * 60; +const MS_IN_ONE_DAY = SEC_IN_ONE_DAY * 1000; + + +var gHttpServer = null; +var gHttpRoot = null; +var gDataRoot = null; +var gPolicy = null; +var gManifestObject = null; +var gManifestHandlerURI = null; + +const TLOG = TELEMETRY_LOG; + +function checkEvent(event, id, data) { + do_print("Checking message " + id); + Assert.equal(event[0], id, "id should match"); + Assert.ok(event[1] > 0, "timestamp should be greater than 0"); + + if (data === undefined) { + Assert.equal(event.length, 2, "event array should have 2 entries"); + } else { + Assert.equal(event.length, data.length + 2, "event entry count should match expected count"); + for (var i = 0; i < data.length; ++i) { + Assert.equal(typeof(event[i + 2]), "string", "event entry should be a string"); + Assert.equal(event[i + 2], data[i], "event entry should match expected entry"); + } + } +} + +function run_test() { + run_next_test(); +} + +add_task(function* test_setup() { + loadAddonManager(); + + gHttpServer = new HttpServer(); + gHttpServer.start(-1); + let port = gHttpServer.identity.primaryPort; + gHttpRoot = "http://localhost:" + port + "/"; + gDataRoot = gHttpRoot + "data/"; + gManifestHandlerURI = gHttpRoot + MANIFEST_HANDLER; + gHttpServer.registerDirectory("/data/", do_get_cwd()); + gHttpServer.registerPathHandler("/" + MANIFEST_HANDLER, (request, response) => { + response.setStatusLine(null, 200, "OK"); + response.write(JSON.stringify(gManifestObject)); + response.processAsync(); + response.finish(); + }); + do_register_cleanup(() => gHttpServer.stop(() => {})); + + Services.prefs.setBoolPref(PREF_EXPERIMENTS_ENABLED, true); + Services.prefs.setIntPref(PREF_LOGGING_LEVEL, 0); + Services.prefs.setBoolPref(PREF_LOGGING_DUMP, true); + Services.prefs.setCharPref(PREF_MANIFEST_URI, gManifestHandlerURI); + Services.prefs.setIntPref(PREF_FETCHINTERVAL, 0); + + gPolicy = new Experiments.Policy(); + let dummyTimer = { cancel: () => {}, clear: () => {} }; + patchPolicy(gPolicy, { + updatechannel: () => "nightly", + oneshotTimer: (callback, timeout, thisObj, name) => dummyTimer, + }); + + yield removeCacheFile(); +}); + +// Test basic starting and stopping of experiments. + +add_task(function* test_telemetryBasics() { + // Check TelemetryLog instead of TelemetrySession.getPayload().log because + // TelemetrySession gets Experiments.instance() and side-effects log entries. + + let expectedLogLength = 0; + + // Dates the following tests are based on. + + let baseDate = new Date(2014, 5, 1, 12); + let startDate1 = futureDate(baseDate, 50 * MS_IN_ONE_DAY); + let endDate1 = futureDate(baseDate, 100 * MS_IN_ONE_DAY); + let startDate2 = futureDate(baseDate, 150 * MS_IN_ONE_DAY); + let endDate2 = futureDate(baseDate, 200 * MS_IN_ONE_DAY); + + // The manifest data we test with. + + gManifestObject = { + "version": 1, + experiments: [ + { + id: EXPERIMENT1_ID, + xpiURL: gDataRoot + EXPERIMENT1_XPI_NAME, + xpiHash: EXPERIMENT1_XPI_SHA1, + startTime: dateToSeconds(startDate1), + endTime: dateToSeconds(endDate1), + maxActiveSeconds: 10 * SEC_IN_ONE_DAY, + appName: ["XPCShell"], + channel: ["nightly"], + }, + { + id: EXPERIMENT2_ID, + xpiURL: gDataRoot + EXPERIMENT2_XPI_NAME, + xpiHash: EXPERIMENT2_XPI_SHA1, + startTime: dateToSeconds(startDate2), + endTime: dateToSeconds(endDate2), + maxActiveSeconds: 10 * SEC_IN_ONE_DAY, + appName: ["XPCShell"], + channel: ["nightly"], + }, + ], + }; + + let experiments = new Experiments.Experiments(gPolicy); + + // Trigger update, clock set to before any activation. + // Use updateManifest() to provide for coverage of that path. + + let now = baseDate; + defineNow(gPolicy, now); + + yield experiments.updateManifest(); + let list = yield experiments.getExperiments(); + Assert.equal(list.length, 0, "Experiment list should be empty."); + + expectedLogLength += 2; + let log = TelemetryLog.entries(); + do_print("Telemetry log: " + JSON.stringify(log)); + Assert.equal(log.length, expectedLogLength, "Telemetry log should have " + expectedLogLength + " entries."); + checkEvent(log[log.length - 2], TLOG.ACTIVATION_KEY, + [TLOG.ACTIVATION.REJECTED, EXPERIMENT1_ID, "startTime"]); + checkEvent(log[log.length - 1], TLOG.ACTIVATION_KEY, + [TLOG.ACTIVATION.REJECTED, EXPERIMENT2_ID, "startTime"]); + + // Trigger update, clock set for experiment 1 to start. + + now = futureDate(startDate1, 5 * MS_IN_ONE_DAY); + defineNow(gPolicy, now); + + yield experiments.updateManifest(); + list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "Experiment list should have 1 entry now."); + + expectedLogLength += 1; + log = TelemetryLog.entries(); + Assert.equal(log.length, expectedLogLength, "Telemetry log should have " + expectedLogLength + " entries. Got " + log.toSource()); + checkEvent(log[log.length - 1], TLOG.ACTIVATION_KEY, + [TLOG.ACTIVATION.ACTIVATED, EXPERIMENT1_ID]); + + // Trigger update, clock set for experiment 1 to stop. + + now = futureDate(endDate1, 1000); + defineNow(gPolicy, now); + + yield experiments.updateManifest(); + list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "Experiment list should have 1 entry."); + + expectedLogLength += 2; + log = TelemetryLog.entries(); + Assert.equal(log.length, expectedLogLength, "Telemetry log should have " + expectedLogLength + " entries."); + checkEvent(log[log.length - 2], TLOG.TERMINATION_KEY, + [TLOG.TERMINATION.EXPIRED, EXPERIMENT1_ID]); + checkEvent(log[log.length - 1], TLOG.ACTIVATION_KEY, + [TLOG.ACTIVATION.REJECTED, EXPERIMENT2_ID, "startTime"]); + + // Trigger update, clock set for experiment 2 to start with invalid hash. + + now = startDate2; + defineNow(gPolicy, now); + gManifestObject.experiments[1].xpiHash = "sha1:0000000000000000000000000000000000000000"; + + yield experiments.updateManifest(); + list = yield experiments.getExperiments(); + Assert.equal(list.length, 1, "Experiment list should have 1 entries."); + + expectedLogLength += 1; + log = TelemetryLog.entries(); + Assert.equal(log.length, expectedLogLength, "Telemetry log should have " + expectedLogLength + " entries."); + checkEvent(log[log.length - 1], TLOG.ACTIVATION_KEY, + [TLOG.ACTIVATION.INSTALL_FAILURE, EXPERIMENT2_ID]); + + // Trigger update, clock set for experiment 2 to properly start now. + + now = futureDate(now, MS_IN_ONE_DAY); + defineNow(gPolicy, now); + gManifestObject.experiments[1].xpiHash = EXPERIMENT2_XPI_SHA1; + + yield experiments.updateManifest(); + list = yield experiments.getExperiments(); + Assert.equal(list.length, 2, "Experiment list should have 2 entries."); + + expectedLogLength += 1; + log = TelemetryLog.entries(); + Assert.equal(log.length, expectedLogLength, "Telemetry log should have " + expectedLogLength + " entries."); + checkEvent(log[log.length - 1], TLOG.ACTIVATION_KEY, + [TLOG.ACTIVATION.ACTIVATED, EXPERIMENT2_ID]); + + // Fake user uninstall of experiment via add-on manager. + + now = futureDate(now, MS_IN_ONE_DAY); + defineNow(gPolicy, now); + + yield experiments.disableExperiment(TLOG.TERMINATION.ADDON_UNINSTALLED); + list = yield experiments.getExperiments(); + Assert.equal(list.length, 2, "Experiment list should have 2 entries."); + + expectedLogLength += 1; + log = TelemetryLog.entries(); + Assert.equal(log.length, expectedLogLength, "Telemetry log should have " + expectedLogLength + " entries."); + checkEvent(log[log.length - 1], TLOG.TERMINATION_KEY, + [TLOG.TERMINATION.ADDON_UNINSTALLED, EXPERIMENT2_ID]); + + // Trigger update with experiment 1a ready to start. + + now = futureDate(now, MS_IN_ONE_DAY); + defineNow(gPolicy, now); + gManifestObject.experiments[0].id = EXPERIMENT3_ID; + gManifestObject.experiments[0].endTime = dateToSeconds(futureDate(now, 50 * MS_IN_ONE_DAY)); + + yield experiments.updateManifest(); + list = yield experiments.getExperiments(); + Assert.equal(list.length, 3, "Experiment list should have 3 entries."); + + expectedLogLength += 1; + log = TelemetryLog.entries(); + Assert.equal(log.length, expectedLogLength, "Telemetry log should have " + expectedLogLength + " entries."); + checkEvent(log[log.length - 1], TLOG.ACTIVATION_KEY, + [TLOG.ACTIVATION.ACTIVATED, EXPERIMENT3_ID]); + + // Trigger disable of an experiment via the API. + + now = futureDate(now, MS_IN_ONE_DAY); + defineNow(gPolicy, now); + + yield experiments.disableExperiment(TLOG.TERMINATION.FROM_API); + list = yield experiments.getExperiments(); + Assert.equal(list.length, 3, "Experiment list should have 3 entries."); + + expectedLogLength += 1; + log = TelemetryLog.entries(); + Assert.equal(log.length, expectedLogLength, "Telemetry log should have " + expectedLogLength + " entries."); + checkEvent(log[log.length - 1], TLOG.TERMINATION_KEY, + [TLOG.TERMINATION.FROM_API, EXPERIMENT3_ID]); + + // Trigger update with experiment 1a ready to start. + + now = futureDate(now, MS_IN_ONE_DAY); + defineNow(gPolicy, now); + gManifestObject.experiments[0].id = EXPERIMENT4_ID; + gManifestObject.experiments[0].endTime = dateToSeconds(futureDate(now, 50 * MS_IN_ONE_DAY)); + + yield experiments.updateManifest(); + list = yield experiments.getExperiments(); + Assert.equal(list.length, 4, "Experiment list should have 4 entries."); + + expectedLogLength += 1; + log = TelemetryLog.entries(); + Assert.equal(log.length, expectedLogLength, "Telemetry log should have " + expectedLogLength + " entries."); + checkEvent(log[log.length - 1], TLOG.ACTIVATION_KEY, + [TLOG.ACTIVATION.ACTIVATED, EXPERIMENT4_ID]); + + // Trigger experiment termination by something other than expiry via the manifest. + + now = futureDate(now, MS_IN_ONE_DAY); + defineNow(gPolicy, now); + gManifestObject.experiments[0].os = "Plan9"; + + yield experiments.updateManifest(); + list = yield experiments.getExperiments(); + Assert.equal(list.length, 4, "Experiment list should have 4 entries."); + + expectedLogLength += 1; + log = TelemetryLog.entries(); + Assert.equal(log.length, expectedLogLength, "Telemetry log should have " + expectedLogLength + " entries."); + checkEvent(log[log.length - 1], TLOG.TERMINATION_KEY, + [TLOG.TERMINATION.RECHECK, EXPERIMENT4_ID, "os"]); + + // Cleanup. + + yield promiseRestartManager(); + yield removeCacheFile(); +}); diff --git a/application/basilisk/experiments/test/xpcshell/test_telemetry_disabled.js b/application/basilisk/experiments/test/xpcshell/test_telemetry_disabled.js new file mode 100644 index 000000000..74f85ccfc --- /dev/null +++ b/application/basilisk/experiments/test/xpcshell/test_telemetry_disabled.js @@ -0,0 +1,21 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; + +Cu.import("resource:///modules/experiments/Experiments.jsm"); + +add_test(function test_experiments_activation() { + do_get_profile(); + loadAddonManager(); + + Services.prefs.setBoolPref(PREF_EXPERIMENTS_ENABLED, true); + Services.prefs.setBoolPref(PREF_TELEMETRY_ENABLED, false); + + let experiments = Experiments.instance(); + Assert.ok(!experiments.enabled, "Experiments must be disabled if Telemetry is disabled."); + + // TODO: Test that Experiments are turned back on when bug 1232648 lands. + + run_next_test(); +}); diff --git a/application/basilisk/experiments/test/xpcshell/test_upgrade.js b/application/basilisk/experiments/test/xpcshell/test_upgrade.js new file mode 100644 index 000000000..f094a406d --- /dev/null +++ b/application/basilisk/experiments/test/xpcshell/test_upgrade.js @@ -0,0 +1,52 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +"use strict"; +Cu.import("resource:///modules/experiments/Experiments.jsm"); + +var cacheData = { + _enabled: true, + _manifestData: { + id: "foobartestid", + xpiURL: "http://example.com/foo.xpi", + xpiHash: "sha256:abcde", + startTime: 0, + endTime: 2000000000, + maxActiveSeconds: 40000000, + appName: "TestApp", + channel: "test-foo", + }, + _needsUpdate: false, + _randomValue: 0.5, + _failedStart: false, + _name: "Foo", + _description: "Foobar", + _homepageURL: "", + _addonId: "foo@test", + _startDate: 0, + _endDate: 2000000000, + _branch: null +}; + +add_task(function* test_valid() { + let e = new Experiments.ExperimentEntry(); + Assert.ok(e.initFromCacheData(cacheData)); + Assert.ok(e.enabled); +}); + +add_task(function* test_upgrade() { + let e = new Experiments.ExperimentEntry(); + delete cacheData._branch; + Assert.ok(e.initFromCacheData(cacheData)); + Assert.ok(e.enabled); +}); + +add_task(function* test_missing() { + let e = new Experiments.ExperimentEntry(); + delete cacheData._name; + Assert.ok(!e.initFromCacheData(cacheData)); +}); + +function run_test() { + run_next_test(); +} diff --git a/application/basilisk/experiments/test/xpcshell/xpcshell.ini b/application/basilisk/experiments/test/xpcshell/xpcshell.ini new file mode 100644 index 000000000..8b13a243f --- /dev/null +++ b/application/basilisk/experiments/test/xpcshell/xpcshell.ini @@ -0,0 +1,30 @@ +[DEFAULT] +head = head.js +tags = addons +firefox-appdir = browser +skip-if = toolkit == 'android' +support-files = + experiments_1.manifest + experiment-1.xpi + experiment-1a.xpi + experiment-2.xpi + experiment-racybranch.xpi + !/toolkit/mozapps/extensions/test/xpcshell/head_addons.js +generated-files = + experiment-1.xpi + experiment-1a.xpi + experiment-2.xpi + experiment-racybranch.xpi + +[test_activate.js] +[test_api.js] +[test_cache.js] +[test_cacherace.js] +[test_conditions.js] +[test_disableExperiments.js] +[test_fetch.js] +[test_telemetry.js] +[test_telemetry_disabled.js] +[test_previous_provider.js] +[test_upgrade.js] +[test_nethang_bug1012924.js] |