diff options
Diffstat (limited to 'application/basilisk/modules/DirectoryLinksProvider.jsm')
| -rw-r--r-- | application/basilisk/modules/DirectoryLinksProvider.jsm | 1243 |
1 files changed, 1243 insertions, 0 deletions
diff --git a/application/basilisk/modules/DirectoryLinksProvider.jsm b/application/basilisk/modules/DirectoryLinksProvider.jsm new file mode 100644 index 000000000..5fe9f9fce --- /dev/null +++ b/application/basilisk/modules/DirectoryLinksProvider.jsm @@ -0,0 +1,1243 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this file, + * You can obtain one at http://mozilla.org/MPL/2.0/. */ + +"use strict"; + +this.EXPORTED_SYMBOLS = ["DirectoryLinksProvider"]; + +const Ci = Components.interfaces; +const Cc = Components.classes; +const Cu = Components.utils; +const ParserUtils = Cc["@mozilla.org/parserutils;1"].getService(Ci.nsIParserUtils); + +Cu.importGlobalProperties(["XMLHttpRequest"]); + +Cu.import("resource://gre/modules/XPCOMUtils.jsm"); +Cu.import("resource://gre/modules/Services.jsm"); +Cu.import("resource://gre/modules/Task.jsm"); +Cu.import("resource://gre/modules/Timer.jsm"); + +XPCOMUtils.defineLazyModuleGetter(this, "NetUtil", + "resource://gre/modules/NetUtil.jsm"); +XPCOMUtils.defineLazyModuleGetter(this, "NewTabUtils", + "resource://gre/modules/NewTabUtils.jsm"); +XPCOMUtils.defineLazyModuleGetter(this, "OS", + "resource://gre/modules/osfile.jsm") +XPCOMUtils.defineLazyModuleGetter(this, "Promise", + "resource://gre/modules/Promise.jsm"); +XPCOMUtils.defineLazyModuleGetter(this, "UpdateUtils", + "resource://gre/modules/UpdateUtils.jsm"); +XPCOMUtils.defineLazyServiceGetter(this, "eTLD", + "@mozilla.org/network/effective-tld-service;1", + "nsIEffectiveTLDService"); +XPCOMUtils.defineLazyGetter(this, "gTextDecoder", () => { + return new TextDecoder(); +}); +XPCOMUtils.defineLazyGetter(this, "gCryptoHash", function() { + return Cc["@mozilla.org/security/hash;1"].createInstance(Ci.nsICryptoHash); +}); +XPCOMUtils.defineLazyGetter(this, "gUnicodeConverter", function() { + let converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"] + .createInstance(Ci.nsIScriptableUnicodeConverter); + converter.charset = "utf8"; + return converter; +}); + + +// The filename where directory links are stored locally +const DIRECTORY_LINKS_FILE = "directoryLinks.json"; +const DIRECTORY_LINKS_TYPE = "application/json"; + +// The preference that tells whether to match the OS locale +const PREF_MATCH_OS_LOCALE = "intl.locale.matchOS"; + +// The preference that tells what locale the user selected +const PREF_SELECTED_LOCALE = "general.useragent.locale"; + +// The preference that tells where to obtain directory links +const PREF_DIRECTORY_SOURCE = "browser.newtabpage.directory.source"; + +// The preference that tells where to send click/view pings +const PREF_DIRECTORY_PING = "browser.newtabpage.directory.ping"; + +// The preference that tells if newtab is enhanced +const PREF_NEWTAB_ENHANCED = "browser.newtabpage.enhanced"; + +// Only allow link urls that are http(s) +const ALLOWED_LINK_SCHEMES = new Set(["http", "https"]); + +// Only allow link image urls that are https or data +const ALLOWED_IMAGE_SCHEMES = new Set(["https", "data"]); + +// Only allow urls to Mozilla's CDN or empty (for data URIs) +const ALLOWED_URL_BASE = new Set(["mozilla.net", ""]); + +// The frecency of a directory link +const DIRECTORY_FRECENCY = 1000; + +// The frecency of a suggested link +const SUGGESTED_FRECENCY = Infinity; + +// The filename where frequency cap data stored locally +const FREQUENCY_CAP_FILE = "frequencyCap.json"; + +// Default settings for daily and total frequency caps +const DEFAULT_DAILY_FREQUENCY_CAP = 3; +const DEFAULT_TOTAL_FREQUENCY_CAP = 10; + +// Default timeDelta to prune unused frequency cap objects +// currently set to 10 days in milliseconds +const DEFAULT_PRUNE_TIME_DELTA = 10 * 24 * 60 * 60 * 1000; + +// The min number of visible (not blocked) history tiles to have before showing suggested tiles +const MIN_VISIBLE_HISTORY_TILES = 8; + +// The max number of visible (not blocked) history tiles to test for inadjacency +const MAX_VISIBLE_HISTORY_TILES = 15; + +// Allowed ping actions remotely stored as columns: case-insensitive [a-z0-9_] +const PING_ACTIONS = ["block", "click", "pin", "sponsored", "sponsored_link", "unpin", "view"]; + +// Location of inadjacent sites json +const INADJACENCY_SOURCE = "chrome://browser/content/newtab/newTab.inadjacent.json"; + +// Fake URL to keep track of last block of a suggested tile in the frequency cap object +const FAKE_SUGGESTED_BLOCK_URL = "ignore://suggested_block"; + +// Time before suggested tile is allowed to play again after block - default to 1 day +const AFTER_SUGGESTED_BLOCK_DECAY_TIME = 24 * 60 * 60 * 1000; + +/** + * Singleton that serves as the provider of directory links. + * Directory links are a hard-coded set of links shown if a user's link + * inventory is empty. + */ +var DirectoryLinksProvider = { + + __linksURL: null, + + _observers: new Set(), + + // links download deferred, resolved upon download completion + _downloadDeferred: null, + + // download default interval is 24 hours in milliseconds + _downloadIntervalMS: 86400000, + + /** + * A mapping from eTLD+1 to an enhanced link objects + */ + _enhancedLinks: new Map(), + + /** + * A mapping from site to a list of suggested link objects + */ + _suggestedLinks: new Map(), + + /** + * Frequency Cap object - maintains daily and total tile counts, and frequency cap settings + */ + _frequencyCaps: {}, + + /** + * A set of top sites that we can provide suggested links for + */ + _topSitesWithSuggestedLinks: new Set(), + + /** + * lookup Set of inadjacent domains + */ + _inadjacentSites: new Set(), + + /** + * This flag is set if there is a suggested tile configured to avoid + * inadjacent sites in new tab + */ + _avoidInadjacentSites: false, + + /** + * This flag is set if _avoidInadjacentSites is true and there is + * an inadjacent site in the new tab + */ + _newTabHasInadjacentSite: false, + + get _observedPrefs() { + return Object.freeze({ + enhanced: PREF_NEWTAB_ENHANCED, + linksURL: PREF_DIRECTORY_SOURCE, + matchOSLocale: PREF_MATCH_OS_LOCALE, + prefSelectedLocale: PREF_SELECTED_LOCALE, + }); + }, + + get _linksURL() { + if (!this.__linksURL) { + try { + this.__linksURL = Services.prefs.getCharPref(this._observedPrefs["linksURL"]); + this.__linksURLModified = Services.prefs.prefHasUserValue(this._observedPrefs["linksURL"]); + } catch (e) { + Cu.reportError("Error fetching directory links url from prefs: " + e); + } + } + return this.__linksURL; + }, + + /** + * Gets the currently selected locale for display. + * @return the selected locale or "en-US" if none is selected + */ + get locale() { + let matchOS; + try { + matchOS = Services.prefs.getBoolPref(PREF_MATCH_OS_LOCALE); + } catch (e) {} + + if (matchOS) { + return Services.locale.getLocaleComponentForUserAgent(); + } + + try { + let locale = Services.prefs.getComplexValue(PREF_SELECTED_LOCALE, + Ci.nsIPrefLocalizedString); + if (locale) { + return locale.data; + } + } catch (e) {} + + try { + return Services.prefs.getCharPref(PREF_SELECTED_LOCALE); + } catch (e) {} + + return "en-US"; + }, + + /** + * Set appropriate default ping behavior controlled by enhanced pref + */ + _setDefaultEnhanced: function DirectoryLinksProvider_setDefaultEnhanced() { + if (!Services.prefs.prefHasUserValue(PREF_NEWTAB_ENHANCED)) { + let enhanced = Services.prefs.getBoolPref(PREF_NEWTAB_ENHANCED); + try { + // Default to not enhanced if DNT is set to tell websites to not track + if (Services.prefs.getBoolPref("privacy.donottrackheader.enabled")) { + enhanced = false; + } + } catch (ex) {} + Services.prefs.setBoolPref(PREF_NEWTAB_ENHANCED, enhanced); + } + }, + + observe: function DirectoryLinksProvider_observe(aSubject, aTopic, aData) { + if (aTopic == "nsPref:changed") { + switch (aData) { + // Re-set the default in case the user clears the pref + case this._observedPrefs.enhanced: + this._setDefaultEnhanced(); + break; + + case this._observedPrefs.linksURL: + delete this.__linksURL; + // fallthrough + + // Force directory download on changes to fetch related prefs + case this._observedPrefs.matchOSLocale: + case this._observedPrefs.prefSelectedLocale: + this._fetchAndCacheLinksIfNecessary(true); + break; + } + } + }, + + _addPrefsObserver: function DirectoryLinksProvider_addObserver() { + for (let pref in this._observedPrefs) { + let prefName = this._observedPrefs[pref]; + Services.prefs.addObserver(prefName, this, false); + } + }, + + _removePrefsObserver: function DirectoryLinksProvider_removeObserver() { + for (let pref in this._observedPrefs) { + let prefName = this._observedPrefs[pref]; + Services.prefs.removeObserver(prefName, this); + } + }, + + _cacheSuggestedLinks(link) { + // Don't cache links that don't have the expected 'frecent_sites' + if (!link.frecent_sites) { + return; + } + + for (let suggestedSite of link.frecent_sites) { + let suggestedMap = this._suggestedLinks.get(suggestedSite) || new Map(); + suggestedMap.set(link.url, link); + this._setupStartEndTime(link); + this._suggestedLinks.set(suggestedSite, suggestedMap); + } + }, + + _fetchAndCacheLinks: function DirectoryLinksProvider_fetchAndCacheLinks(uri) { + // Replace with the same display locale used for selecting links data + uri = uri.replace("%LOCALE%", this.locale); + uri = uri.replace("%CHANNEL%", UpdateUtils.UpdateChannel); + + return this._downloadJsonData(uri).then(json => { + return OS.File.writeAtomic(this._directoryFilePath, json, {tmpPath: this._directoryFilePath + ".tmp"}); + }); + }, + + /** + * Downloads a links with json content + * @param download uri + * @return promise resolved to json string, "{}" returned if status != 200 + */ + _downloadJsonData: function DirectoryLinksProvider__downloadJsonData(uri) { + let deferred = Promise.defer(); + let xmlHttp = this._newXHR(); + + xmlHttp.onload = function(aResponse) { + let json = this.responseText; + if (this.status && this.status != 200) { + json = "{}"; + } + deferred.resolve(json); + }; + + xmlHttp.onerror = function(e) { + deferred.reject("Fetching " + uri + " results in error code: " + e.target.status); + }; + + try { + xmlHttp.open("GET", uri); + // Override the type so XHR doesn't complain about not well-formed XML + xmlHttp.overrideMimeType(DIRECTORY_LINKS_TYPE); + // Set the appropriate request type for servers that require correct types + xmlHttp.setRequestHeader("Content-Type", DIRECTORY_LINKS_TYPE); + xmlHttp.send(); + } catch (e) { + deferred.reject("Error fetching " + uri); + Cu.reportError(e); + } + return deferred.promise; + }, + + /** + * Downloads directory links if needed + * @return promise resolved immediately if no download needed, or upon completion + */ + _fetchAndCacheLinksIfNecessary: function DirectoryLinksProvider_fetchAndCacheLinksIfNecessary(forceDownload = false) { + if (this._downloadDeferred) { + // fetching links already - just return the promise + return this._downloadDeferred.promise; + } + + if (forceDownload || this._needsDownload) { + this._downloadDeferred = Promise.defer(); + this._fetchAndCacheLinks(this._linksURL).then(() => { + // the new file was successfully downloaded and cached, so update a timestamp + this._lastDownloadMS = Date.now(); + this._downloadDeferred.resolve(); + this._downloadDeferred = null; + this._callObservers("onManyLinksChanged") + }, + error => { + this._downloadDeferred.resolve(); + this._downloadDeferred = null; + this._callObservers("onDownloadFail"); + }); + return this._downloadDeferred.promise; + } + + // download is not needed + return Promise.resolve(); + }, + + /** + * @return true if download is needed, false otherwise + */ + get _needsDownload() { + // fail if last download occured less then 24 hours ago + if ((Date.now() - this._lastDownloadMS) > this._downloadIntervalMS) { + return true; + } + return false; + }, + + /** + * Create a new XMLHttpRequest that is anonymous, i.e., doesn't send cookies + */ + _newXHR() { + return new XMLHttpRequest({mozAnon: true}); + }, + + /** + * Reads directory links file and parses its content + * @return a promise resolved to an object with keys 'directory' and 'suggested', + * each containing a valid list of links, + * or {'directory': [], 'suggested': []} if read or parse fails. + */ + _readDirectoryLinksFile: function DirectoryLinksProvider_readDirectoryLinksFile() { + let emptyOutput = {directory: [], suggested: [], enhanced: []}; + return OS.File.read(this._directoryFilePath).then(binaryData => { + let output; + try { + let json = gTextDecoder.decode(binaryData); + let linksObj = JSON.parse(json); + output = {directory: linksObj.directory || [], + suggested: linksObj.suggested || [], + enhanced: linksObj.enhanced || []}; + } catch (e) { + Cu.reportError(e); + } + return output || emptyOutput; + }, + error => { + Cu.reportError(error); + return emptyOutput; + }); + }, + + /** + * Translates link.time_limits to UTC miliseconds and sets + * link.startTime and link.endTime properties in link object + */ + _setupStartEndTime: function DirectoryLinksProvider_setupStartEndTime(link) { + // set start/end limits. Use ISO_8601 format: '2014-01-10T20:20:20.600Z' + // (details here http://en.wikipedia.org/wiki/ISO_8601) + // Note that if timezone is missing, FX will interpret as local time + // meaning that the server can sepecify any time, but if the capmaign + // needs to start at same time across multiple timezones, the server + // omits timezone indicator + if (!link.time_limits) { + return; + } + + let parsedTime; + if (link.time_limits.start) { + parsedTime = Date.parse(link.time_limits.start); + if (parsedTime && !isNaN(parsedTime)) { + link.startTime = parsedTime; + } + } + if (link.time_limits.end) { + parsedTime = Date.parse(link.time_limits.end); + if (parsedTime && !isNaN(parsedTime)) { + link.endTime = parsedTime; + } + } + }, + + /* + * Handles campaign timeout + */ + _onCampaignTimeout: function DirectoryLinksProvider_onCampaignTimeout() { + // _campaignTimeoutID is invalid here, so just set it to null + this._campaignTimeoutID = null; + this._updateSuggestedTile(); + }, + + /* + * Clears capmpaign timeout + */ + _clearCampaignTimeout: function DirectoryLinksProvider_clearCampaignTimeout() { + if (this._campaignTimeoutID) { + clearTimeout(this._campaignTimeoutID); + this._campaignTimeoutID = null; + } + }, + + /** + * Setup capmpaign timeout to recompute suggested tiles upon + * reaching soonest start or end time for the campaign + * @param timeout in milliseconds + */ + _setupCampaignTimeCheck: function DirectoryLinksProvider_setupCampaignTimeCheck(timeout) { + // sanity check + if (!timeout || timeout <= 0) { + return; + } + this._clearCampaignTimeout(); + // setup next timeout + this._campaignTimeoutID = setTimeout(this._onCampaignTimeout.bind(this), timeout); + }, + + /** + * Test link for campaign time limits: checks if link falls within start/end time + * and returns an object containing a use flag and the timeoutDate milliseconds + * when the link has to be re-checked for campaign start-ready or end-reach + * @param link + * @return object {use: true or false, timeoutDate: milliseconds or null} + */ + _testLinkForCampaignTimeLimits: function DirectoryLinksProvider_testLinkForCampaignTimeLimits(link) { + let currentTime = Date.now(); + // test for start time first + if (link.startTime && link.startTime > currentTime) { + // not yet ready for start + return {use: false, timeoutDate: link.startTime}; + } + // otherwise check for end time + if (link.endTime) { + // passed end time + if (link.endTime <= currentTime) { + return {use: false}; + } + // otherwise link is still ok, but we need to set timeoutDate + return {use: true, timeoutDate: link.endTime}; + } + // if we are here, the link is ok and no timeoutDate needed + return {use: true}; + }, + + /** + * Handles block on suggested tile: updates fake block url with current timestamp + */ + handleSuggestedTileBlock: function DirectoryLinksProvider_handleSuggestedTileBlock() { + this._updateFrequencyCapSettings({url: FAKE_SUGGESTED_BLOCK_URL}); + this._writeFrequencyCapFile(); + this._updateSuggestedTile(); + }, + + /** + * Checks if suggested tile is being blocked for the rest of "decay time" + * @return True if blocked, false otherwise + */ + _isSuggestedTileBlocked: function DirectoryLinksProvider__isSuggestedTileBlocked() { + let capObject = this._frequencyCaps[FAKE_SUGGESTED_BLOCK_URL]; + if (!capObject || !capObject.lastUpdated) { + // user never blocked suggested tile or lastUpdated is missing + return false; + } + // otherwise, make sure that enough time passed after suggested tile was blocked + return (capObject.lastUpdated + AFTER_SUGGESTED_BLOCK_DECAY_TIME) > Date.now(); + }, + + /** + * Report some action on a newtab page (view, click) + * @param sites Array of sites shown on newtab page + * @param action String of the behavior to report + * @param triggeringSiteIndex optional Int index of the site triggering action + * @return download promise + */ + reportSitesAction: function DirectoryLinksProvider_reportSitesAction(sites, action, triggeringSiteIndex) { + // Check if the suggested tile was shown + if (action == "view") { + sites.slice(0, triggeringSiteIndex + 1).filter(s => s).forEach(site => { + let {targetedSite, url} = site.link; + if (targetedSite) { + this._addFrequencyCapView(url); + } + }); + } else { + // any click action on a suggested tile should stop that tile suggestion + // click/block - user either removed a tile or went to a landing page + // pin - tile turned into history tile, should no longer be suggested + // unpin - the tile was pinned before, should not matter + // suggested tile has targetedSite, or frecent_sites if it was pinned + let {frecent_sites, targetedSite, url} = sites[triggeringSiteIndex].link; + if (frecent_sites || targetedSite) { + this._setFrequencyCapClick(url); + } + } + + let newtabEnhanced = false; + let pingEndPoint = ""; + try { + newtabEnhanced = Services.prefs.getBoolPref(PREF_NEWTAB_ENHANCED); + pingEndPoint = Services.prefs.getCharPref(PREF_DIRECTORY_PING); + } catch (ex) {} + + // Bug 1240245 - We no longer send pings, but frequency capping and fetching + // tests depend on the following actions, so references to PING remain. + let invalidAction = PING_ACTIONS.indexOf(action) == -1; + if (!newtabEnhanced || pingEndPoint == "" || invalidAction) { + return Promise.resolve(); + } + + return Task.spawn(function* () { + // since we updated views/clicks we need write _frequencyCaps to disk + yield this._writeFrequencyCapFile(); + // Use this as an opportunity to potentially fetch new links + yield this._fetchAndCacheLinksIfNecessary(); + }.bind(this)); + }, + + /** + * Get the enhanced link object for a link (whether history or directory) + */ + getEnhancedLink: function DirectoryLinksProvider_getEnhancedLink(link) { + // Use the provided link if it's already enhanced + return link.enhancedImageURI && link ? link : + this._enhancedLinks.get(NewTabUtils.extractSite(link.url)); + }, + + /** + * Check if a url's scheme is in a Set of allowed schemes and if the base + * domain is allowed. + * @param url to check + * @param allowed Set of allowed schemes + * @param checkBase boolean to check the base domain + */ + isURLAllowed(url, allowed, checkBase) { + // Assume no url is an allowed url + if (!url) { + return true; + } + + let scheme = "", base = ""; + try { + // A malformed url will not be allowed + let uri = Services.io.newURI(url); + scheme = uri.scheme; + + // URIs without base domains will be allowed + base = Services.eTLD.getBaseDomain(uri); + } catch (ex) {} + // Require a scheme match and the base only if desired + return allowed.has(scheme) && (!checkBase || ALLOWED_URL_BASE.has(base)); + }, + + _escapeChars(text) { + let charMap = { + "&": "&", + "<": "<", + ">": ">", + '"': """, + "'": "'" + }; + + return text.replace(/[&<>"']/g, (character) => charMap[character]); + }, + + /** + * Gets the current set of directory links. + * @param aCallback The function that the array of links is passed to. + */ + getLinks: function DirectoryLinksProvider_getLinks(aCallback) { + this._readDirectoryLinksFile().then(rawLinks => { + // Reset the cache of suggested tiles and enhanced images for this new set of links + this._enhancedLinks.clear(); + this._suggestedLinks.clear(); + this._clearCampaignTimeout(); + this._avoidInadjacentSites = false; + + // Only check base domain for images when using the default pref + let checkBase = !this.__linksURLModified; + let validityFilter = function(link) { + // Make sure the link url is allowed and images too if they exist + return this.isURLAllowed(link.url, ALLOWED_LINK_SCHEMES, false) && + (!link.imageURI || + this.isURLAllowed(link.imageURI, ALLOWED_IMAGE_SCHEMES, checkBase)) && + (!link.enhancedImageURI || + this.isURLAllowed(link.enhancedImageURI, ALLOWED_IMAGE_SCHEMES, checkBase)); + }.bind(this); + + rawLinks.suggested.filter(validityFilter).forEach((link, position) => { + // Suggested sites must have an adgroup name. + if (!link.adgroup_name) { + return; + } + + let sanitizeFlags = ParserUtils.SanitizerCidEmbedsOnly | + ParserUtils.SanitizerDropForms | + ParserUtils.SanitizerDropNonCSSPresentation; + + link.explanation = this._escapeChars(link.explanation ? ParserUtils.convertToPlainText(link.explanation, sanitizeFlags, 0) : ""); + link.targetedName = this._escapeChars(ParserUtils.convertToPlainText(link.adgroup_name, sanitizeFlags, 0)); + link.lastVisitDate = rawLinks.suggested.length - position; + // check if link wants to avoid inadjacent sites + if (link.check_inadjacency) { + this._avoidInadjacentSites = true; + } + + // We cache suggested tiles here but do not push any of them in the links list yet. + // The decision for which suggested tile to include will be made separately. + this._cacheSuggestedLinks(link); + this._updateFrequencyCapSettings(link); + }); + + rawLinks.enhanced.filter(validityFilter).forEach((link, position) => { + link.lastVisitDate = rawLinks.enhanced.length - position; + + // Stash the enhanced image for the site + if (link.enhancedImageURI) { + this._enhancedLinks.set(NewTabUtils.extractSite(link.url), link); + } + }); + + let links = rawLinks.directory.filter(validityFilter).map((link, position) => { + link.lastVisitDate = rawLinks.directory.length - position; + link.frecency = DIRECTORY_FRECENCY; + return link; + }); + + // Allow for one link suggestion on top of the default directory links + this.maxNumLinks = links.length + 1; + + // prune frequency caps of outdated urls + this._pruneFrequencyCapUrls(); + // write frequency caps object to disk asynchronously + this._writeFrequencyCapFile(); + + return links; + }).catch(ex => { + Cu.reportError(ex); + return []; + }).then(links => { + aCallback(links); + this._populatePlacesLinks(); + }); + }, + + init: function DirectoryLinksProvider_init() { + this._setDefaultEnhanced(); + this._addPrefsObserver(); + // setup directory file path and last download timestamp + this._directoryFilePath = OS.Path.join(OS.Constants.Path.localProfileDir, DIRECTORY_LINKS_FILE); + this._lastDownloadMS = 0; + + // setup frequency cap file path + this._frequencyCapFilePath = OS.Path.join(OS.Constants.Path.localProfileDir, FREQUENCY_CAP_FILE); + // setup inadjacent sites URL + this._inadjacentSitesUrl = INADJACENCY_SOURCE; + + NewTabUtils.placesProvider.addObserver(this); + NewTabUtils.links.addObserver(this); + + return Task.spawn(function*() { + // get the last modified time of the links file if it exists + let doesFileExists = yield OS.File.exists(this._directoryFilePath); + if (doesFileExists) { + let fileInfo = yield OS.File.stat(this._directoryFilePath); + this._lastDownloadMS = Date.parse(fileInfo.lastModificationDate); + } + // read frequency cap file + yield this._readFrequencyCapFile(); + // fetch directory on startup without force + yield this._fetchAndCacheLinksIfNecessary(); + // fecth inadjacent sites on startup + yield this._loadInadjacentSites(); + }.bind(this)); + }, + + _handleManyLinksChanged() { + this._topSitesWithSuggestedLinks.clear(); + this._suggestedLinks.forEach((suggestedLinks, site) => { + if (NewTabUtils.isTopPlacesSite(site)) { + this._topSitesWithSuggestedLinks.add(site); + } + }); + this._updateSuggestedTile(); + }, + + /** + * Updates _topSitesWithSuggestedLinks based on the link that was changed. + * + * @return true if _topSitesWithSuggestedLinks was modified, false otherwise. + */ + _handleLinkChanged(aLink) { + let changedLinkSite = NewTabUtils.extractSite(aLink.url); + let linkStored = this._topSitesWithSuggestedLinks.has(changedLinkSite); + + if (!NewTabUtils.isTopPlacesSite(changedLinkSite) && linkStored) { + this._topSitesWithSuggestedLinks.delete(changedLinkSite); + return true; + } + + if (this._suggestedLinks.has(changedLinkSite) && + NewTabUtils.isTopPlacesSite(changedLinkSite) && !linkStored) { + this._topSitesWithSuggestedLinks.add(changedLinkSite); + return true; + } + + // always run _updateSuggestedTile if aLink is inadjacent + // and there are tiles configured to avoid it + if (this._avoidInadjacentSites && this._isInadjacentLink(aLink)) { + return true; + } + + return false; + }, + + _populatePlacesLinks() { + NewTabUtils.links.populateProviderCache(NewTabUtils.placesProvider, () => { + this._handleManyLinksChanged(); + }); + }, + + onDeleteURI(aProvider, aLink) { + let {url} = aLink; + // remove clicked flag for that url and + // call observer upon disk write completion + this._removeTileClick(url).then(() => { + this._callObservers("onDeleteURI", url); + }); + }, + + onClearHistory() { + // remove all clicked flags and call observers upon file write + this._removeAllTileClicks().then(() => { + this._callObservers("onClearHistory"); + }); + }, + + onLinkChanged(aProvider, aLink) { + // Make sure NewTabUtils.links handles the notification first. + setTimeout(() => { + if (this._handleLinkChanged(aLink) || this._shouldUpdateSuggestedTile()) { + this._updateSuggestedTile(); + } + }, 0); + }, + + onManyLinksChanged() { + // Make sure NewTabUtils.links handles the notification first. + setTimeout(() => { + this._handleManyLinksChanged(); + }, 0); + }, + + _getCurrentTopSiteCount() { + let visibleTopSiteCount = 0; + let newTabLinks = NewTabUtils.links.getLinks(); + for (let link of newTabLinks.slice(0, MIN_VISIBLE_HISTORY_TILES)) { + // compute visibleTopSiteCount for suggested tiles + if (link && (link.type == "history" || link.type == "enhanced")) { + visibleTopSiteCount++; + } + } + // since newTabLinks are available, set _newTabHasInadjacentSite here + // note that _shouldUpdateSuggestedTile is called by _updateSuggestedTile + this._newTabHasInadjacentSite = this._avoidInadjacentSites && this._checkForInadjacentSites(newTabLinks); + + return visibleTopSiteCount; + }, + + _shouldUpdateSuggestedTile() { + let sortedLinks = NewTabUtils.getProviderLinks(this); + + let mostFrecentLink = {}; + if (sortedLinks && sortedLinks.length) { + mostFrecentLink = sortedLinks[0] + } + + let currTopSiteCount = this._getCurrentTopSiteCount(); + if ((!mostFrecentLink.targetedSite && currTopSiteCount >= MIN_VISIBLE_HISTORY_TILES) || + (mostFrecentLink.targetedSite && currTopSiteCount < MIN_VISIBLE_HISTORY_TILES)) { + // If mostFrecentLink has a targetedSite then mostFrecentLink is a suggested link. + // If we have enough history links (8+) to show a suggested tile and we are not + // already showing one, then we should update (to *attempt* to add a suggested tile). + // OR if we don't have enough history to show a suggested tile (<8) and we are + // currently showing one, we should update (to remove it). + return true; + } + + return false; + }, + + /** + * Chooses and returns a suggested tile based on a user's top sites + * that we have an available suggested tile for. + * + * @return the chosen suggested tile, or undefined if there isn't one + */ + _updateSuggestedTile() { + let sortedLinks = NewTabUtils.getProviderLinks(this); + + if (!sortedLinks) { + // If NewTabUtils.links.resetCache() is called before getting here, + // sortedLinks may be undefined. + return undefined; + } + + // Delete the current suggested tile, if one exists. + let initialLength = sortedLinks.length; + if (initialLength) { + let mostFrecentLink = sortedLinks[0]; + if (mostFrecentLink.targetedSite) { + this._callObservers("onLinkChanged", { + url: mostFrecentLink.url, + frecency: SUGGESTED_FRECENCY, + lastVisitDate: mostFrecentLink.lastVisitDate, + type: mostFrecentLink.type, + }, 0, true); + } + } + + if (this._topSitesWithSuggestedLinks.size == 0 || + !this._shouldUpdateSuggestedTile() || + this._isSuggestedTileBlocked()) { + // There are no potential suggested links we can show or not + // enough history for a suggested tile, or suggested tile was + // recently blocked and wait time interval has not decayed yet + return undefined; + } + + // Create a flat list of all possible links we can show as suggested. + // Note that many top sites may map to the same suggested links, but we only + // want to count each suggested link once (based on url), thus possibleLinks is a map + // from url to suggestedLink. Thus, each link has an equal chance of being chosen at + // random from flattenedLinks if it appears only once. + let nextTimeout; + let possibleLinks = new Map(); + let targetedSites = new Map(); + this._topSitesWithSuggestedLinks.forEach(topSiteWithSuggestedLink => { + let suggestedLinksMap = this._suggestedLinks.get(topSiteWithSuggestedLink); + suggestedLinksMap.forEach((suggestedLink, url) => { + // Skip this link if we've shown it too many times already + if (!this._testFrequencyCapLimits(url)) { + return; + } + + // as we iterate suggestedLinks, check for campaign start/end + // time limits, and set nextTimeout to the closest timestamp + let {use, timeoutDate} = this._testLinkForCampaignTimeLimits(suggestedLink); + // update nextTimeout is necessary + if (timeoutDate && (!nextTimeout || nextTimeout > timeoutDate)) { + nextTimeout = timeoutDate; + } + // Skip link if it falls outside campaign time limits + if (!use) { + return; + } + + // Skip link if it avoids inadjacent sites and newtab has one + if (suggestedLink.check_inadjacency && this._newTabHasInadjacentSite) { + return; + } + + possibleLinks.set(url, suggestedLink); + + // Keep a map of URL to targeted sites. We later use this to show the user + // what site they visited to trigger this suggestion. + if (!targetedSites.get(url)) { + targetedSites.set(url, []); + } + targetedSites.get(url).push(topSiteWithSuggestedLink); + }) + }); + + // setup timeout check for starting or ending campaigns + if (nextTimeout) { + this._setupCampaignTimeCheck(nextTimeout - Date.now()); + } + + // We might have run out of possible links to show + let numLinks = possibleLinks.size; + if (numLinks == 0) { + return undefined; + } + + let flattenedLinks = [...possibleLinks.values()]; + + // Choose our suggested link at random + let suggestedIndex = Math.floor(Math.random() * numLinks); + let chosenSuggestedLink = flattenedLinks[suggestedIndex]; + + // Add the suggested link to the front with some extra values + this._callObservers("onLinkChanged", Object.assign({ + frecency: SUGGESTED_FRECENCY, + + // Choose the first site a user has visited as the target. In the future, + // this should be the site with the highest frecency. However, we currently + // store frecency by URL not by site. + targetedSite: targetedSites.get(chosenSuggestedLink.url).length ? + targetedSites.get(chosenSuggestedLink.url)[0] : null + }, chosenSuggestedLink)); + return chosenSuggestedLink; + }, + + /** + * Loads inadjacent sites + * @return a promise resolved when lookup Set for sites is built + */ + _loadInadjacentSites: function DirectoryLinksProvider_loadInadjacentSites() { + return this._downloadJsonData(this._inadjacentSitesUrl).then(jsonString => { + let jsonObject = {}; + try { + jsonObject = JSON.parse(jsonString); + } catch (e) { + Cu.reportError(e); + } + + this._inadjacentSites = new Set(jsonObject.domains); + }); + }, + + /** + * Genegrates hash suitable for looking up inadjacent site + * @param value to hsh + * @return hased value, base64-ed + */ + _generateHash: function DirectoryLinksProvider_generateHash(value) { + let byteArr = gUnicodeConverter.convertToByteArray(value); + gCryptoHash.init(gCryptoHash.MD5); + gCryptoHash.update(byteArr, byteArr.length); + return gCryptoHash.finish(true); + }, + + /** + * Checks if link belongs to inadjacent domain + * @param link to check + * @return true for inadjacent domains, false otherwise + */ + _isInadjacentLink: function DirectoryLinksProvider_isInadjacentLink(link) { + let baseDomain = link.baseDomain || NewTabUtils.extractSite(link.url || ""); + if (!baseDomain) { + return false; + } + // check if hashed domain is inadjacent + return this._inadjacentSites.has(this._generateHash(baseDomain)); + }, + + /** + * Checks if new tab has inadjacent site + * @param new tab links (or nothing, in which case NewTabUtils.links.getLinks() is called + * @return true if new tab shows has inadjacent site + */ + _checkForInadjacentSites: function DirectoryLinksProvider_checkForInadjacentSites(newTabLink) { + let links = newTabLink || NewTabUtils.links.getLinks(); + for (let link of links.slice(0, MAX_VISIBLE_HISTORY_TILES)) { + // check links against inadjacent list - specifically include ALL link types + if (this._isInadjacentLink(link)) { + return true; + } + } + return false; + }, + + /** + * Reads json file, parses its content, and returns resulting object + * @param json file path + * @param json object to return in case file read or parse fails + * @return a promise resolved to a valid object or undefined upon error + */ + _readJsonFile: Task.async(function* (filePath, nullObject) { + let jsonObj; + try { + let binaryData = yield OS.File.read(filePath); + let json = gTextDecoder.decode(binaryData); + jsonObj = JSON.parse(json); + } catch (e) {} + return jsonObj || nullObject; + }), + + /** + * Loads frequency cap object from file and parses its content + * @return a promise resolved upon load completion + * on error or non-exstent file _frequencyCaps is set to empty object + */ + _readFrequencyCapFile: Task.async(function* () { + // set _frequencyCaps object to file's content or empty object + this._frequencyCaps = yield this._readJsonFile(this._frequencyCapFilePath, {}); + }), + + /** + * Saves frequency cap object to file + * @return a promise resolved upon file i/o completion + */ + _writeFrequencyCapFile: function DirectoryLinksProvider_writeFrequencyCapFile() { + let json = JSON.stringify(this._frequencyCaps || {}); + return OS.File.writeAtomic(this._frequencyCapFilePath, json, {tmpPath: this._frequencyCapFilePath + ".tmp"}); + }, + + /** + * Clears frequency cap object and writes empty json to file + * @return a promise resolved upon file i/o completion + */ + _clearFrequencyCap: function DirectoryLinksProvider_clearFrequencyCap() { + this._frequencyCaps = {}; + return this._writeFrequencyCapFile(); + }, + + /** + * updates frequency cap configuration for a link + */ + _updateFrequencyCapSettings: function DirectoryLinksProvider_updateFrequencyCapSettings(link) { + let capsObject = this._frequencyCaps[link.url]; + if (!capsObject) { + // create an object with empty counts + capsObject = { + dailyViews: 0, + totalViews: 0, + lastShownDate: 0, + }; + this._frequencyCaps[link.url] = capsObject; + } + // set last updated timestamp + capsObject.lastUpdated = Date.now(); + // check for link configuration + if (link.frequency_caps) { + capsObject.dailyCap = link.frequency_caps.daily || DEFAULT_DAILY_FREQUENCY_CAP; + capsObject.totalCap = link.frequency_caps.total || DEFAULT_TOTAL_FREQUENCY_CAP; + } else { + // fallback to defaults + capsObject.dailyCap = DEFAULT_DAILY_FREQUENCY_CAP; + capsObject.totalCap = DEFAULT_TOTAL_FREQUENCY_CAP; + } + }, + + /** + * Prunes frequency cap objects for outdated links + * @param timeDetla milliseconds + * all cap objects with lastUpdated less than (now() - timeDelta) + * will be removed. This is done to remove frequency cap objects + * for unused tile urls + */ + _pruneFrequencyCapUrls: function DirectoryLinksProvider_pruneFrequencyCapUrls(timeDelta = DEFAULT_PRUNE_TIME_DELTA) { + let timeThreshold = Date.now() - timeDelta; + Object.keys(this._frequencyCaps).forEach(url => { + // remove url if it is not ignorable and wasn't updated for a while + if (!url.startsWith("ignore") && this._frequencyCaps[url].lastUpdated <= timeThreshold) { + delete this._frequencyCaps[url]; + } + }); + }, + + /** + * Checks if supplied timestamp happened today + * @param timestamp in milliseconds + * @return true if the timestamp was made today, false otherwise + */ + _wasToday: function DirectoryLinksProvider_wasToday(timestamp) { + let showOn = new Date(timestamp); + let today = new Date(); + // call timestamps identical if both day and month are same + return showOn.getDate() == today.getDate() && + showOn.getMonth() == today.getMonth() && + showOn.getYear() == today.getYear(); + }, + + /** + * adds some number of views for a url + * @param url String url of the suggested link + */ + _addFrequencyCapView: function DirectoryLinksProvider_addFrequencyCapView(url) { + let capObject = this._frequencyCaps[url]; + // sanity check + if (!capObject) { + return; + } + + // if the day is new: reset the daily counter and lastShownDate + if (!this._wasToday(capObject.lastShownDate)) { + capObject.dailyViews = 0; + // update lastShownDate + capObject.lastShownDate = Date.now(); + } + + // bump both daily and total counters + capObject.totalViews++; + capObject.dailyViews++; + + // if any of the caps is reached - update suggested tiles + if (capObject.totalViews >= capObject.totalCap || + capObject.dailyViews >= capObject.dailyCap) { + this._updateSuggestedTile(); + } + }, + + /** + * Sets clicked flag for link url + * @param url String url of the suggested link + */ + _setFrequencyCapClick(url) { + let capObject = this._frequencyCaps[url]; + // sanity check + if (!capObject) { + return; + } + capObject.clicked = true; + // and update suggested tiles, since current tile became invalid + this._updateSuggestedTile(); + }, + + /** + * Tests frequency cap limits for link url + * @param url String url of the suggested link + * @return true if link is viewable, false otherwise + */ + _testFrequencyCapLimits: function DirectoryLinksProvider_testFrequencyCapLimits(url) { + let capObject = this._frequencyCaps[url]; + // sanity check: if url is missing - do not show this tile + if (!capObject) { + return false; + } + + // check for clicked set or total views reached + if (capObject.clicked || capObject.totalViews >= capObject.totalCap) { + return false; + } + + // otherwise check if link is over daily views limit + if (this._wasToday(capObject.lastShownDate) && + capObject.dailyViews >= capObject.dailyCap) { + return false; + } + + // we passed all cap tests: return true + return true; + }, + + /** + * Removes clicked flag from frequency cap entry for tile landing url + * @param url String url of the suggested link + * @return promise resolved upon disk write completion + */ + _removeTileClick: function DirectoryLinksProvider_removeTileClick(url = "") { + // remove trailing slash, to accomodate Places sending site urls ending with '/' + let noTrailingSlashUrl = url.replace(/\/$/, ""); + let capObject = this._frequencyCaps[url] || this._frequencyCaps[noTrailingSlashUrl]; + // return resolved promise if capObject is not found + if (!capObject) { + return Promise.resolve(); + } + // otherwise remove clicked flag + delete capObject.clicked; + return this._writeFrequencyCapFile(); + }, + + /** + * Removes all clicked flags from frequency cap object + * @return promise resolved upon disk write completion + */ + _removeAllTileClicks: function DirectoryLinksProvider_removeAllTileClicks() { + Object.keys(this._frequencyCaps).forEach(url => { + delete this._frequencyCaps[url].clicked; + }); + return this._writeFrequencyCapFile(); + }, + + /** + * Return the object to its pre-init state + */ + reset: function DirectoryLinksProvider_reset() { + delete this.__linksURL; + this._removePrefsObserver(); + this._removeObservers(); + }, + + addObserver: function DirectoryLinksProvider_addObserver(aObserver) { + this._observers.add(aObserver); + }, + + removeObserver: function DirectoryLinksProvider_removeObserver(aObserver) { + this._observers.delete(aObserver); + }, + + _callObservers(methodName, ...args) { + for (let obs of this._observers) { + if (typeof(obs[methodName]) == "function") { + try { + obs[methodName](this, ...args); + } catch (err) { + Cu.reportError(err); + } + } + } + }, + + _removeObservers() { + this._observers.clear(); + } +}; |