diff options
Diffstat (limited to 'toolkit/components/search/current/nsSearchService.js')
| -rw-r--r-- | toolkit/components/search/current/nsSearchService.js | 4780 |
1 files changed, 4780 insertions, 0 deletions
diff --git a/toolkit/components/search/current/nsSearchService.js b/toolkit/components/search/current/nsSearchService.js new file mode 100644 index 000000000..9f9003516 --- /dev/null +++ b/toolkit/components/search/current/nsSearchService.js @@ -0,0 +1,4780 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +const Ci = Components.interfaces; +const Cc = Components.classes; +const Cr = Components.results; +const Cu = Components.utils; + +Cu.import("resource://gre/modules/XPCOMUtils.jsm"); +Cu.import("resource://gre/modules/Services.jsm"); +Cu.import("resource://gre/modules/Promise.jsm"); +Cu.import("resource://gre/modules/debug.js"); +Cu.import("resource://gre/modules/AppConstants.jsm"); + +XPCOMUtils.defineLazyModuleGetter(this, "AsyncShutdown", + "resource://gre/modules/AsyncShutdown.jsm"); +XPCOMUtils.defineLazyModuleGetter(this, "DeferredTask", + "resource://gre/modules/DeferredTask.jsm"); +XPCOMUtils.defineLazyModuleGetter(this, "OS", + "resource://gre/modules/osfile.jsm"); +XPCOMUtils.defineLazyModuleGetter(this, "Task", + "resource://gre/modules/Task.jsm"); +XPCOMUtils.defineLazyModuleGetter(this, "TelemetryStopwatch", + "resource://gre/modules/TelemetryStopwatch.jsm"); +XPCOMUtils.defineLazyModuleGetter(this, "SearchStaticData", + "resource://gre/modules/SearchStaticData.jsm"); +XPCOMUtils.defineLazyModuleGetter(this, "setTimeout", + "resource://gre/modules/Timer.jsm"); +XPCOMUtils.defineLazyModuleGetter(this, "clearTimeout", + "resource://gre/modules/Timer.jsm"); +XPCOMUtils.defineLazyModuleGetter(this, "Lz4", + "resource://gre/modules/lz4.js"); + +XPCOMUtils.defineLazyServiceGetter(this, "gTextToSubURI", + "@mozilla.org/intl/texttosuburi;1", + "nsITextToSubURI"); +XPCOMUtils.defineLazyServiceGetter(this, "gEnvironment", + "@mozilla.org/process/environment;1", + "nsIEnvironment"); + +Cu.importGlobalProperties(["XMLHttpRequest"]); + +// A text encoder to UTF8, used whenever we commit the cache to disk. +XPCOMUtils.defineLazyGetter(this, "gEncoder", + function() { + return new TextEncoder(); + }); + +const MODE_RDONLY = 0x01; +const MODE_WRONLY = 0x02; +const MODE_CREATE = 0x08; +const MODE_APPEND = 0x10; +const MODE_TRUNCATE = 0x20; + +// Directory service keys +const NS_APP_SEARCH_DIR_LIST = "SrchPluginsDL"; +const NS_APP_DISTRIBUTION_SEARCH_DIR_LIST = "SrchPluginsDistDL"; +const NS_APP_USER_SEARCH_DIR = "UsrSrchPlugns"; +const NS_APP_SEARCH_DIR = "SrchPlugns"; +const NS_APP_USER_PROFILE_50_DIR = "ProfD"; + +// Loading plugins from NS_APP_SEARCH_DIR is no longer supported. +// Instead, we now load plugins from APP_SEARCH_PREFIX, where a +// list.txt file needs to exist to list available engines. +const APP_SEARCH_PREFIX = "resource://search-plugins/"; + +// See documentation in nsIBrowserSearchService.idl. +const SEARCH_ENGINE_TOPIC = "browser-search-engine-modified"; +const QUIT_APPLICATION_TOPIC = "quit-application"; + +const SEARCH_ENGINE_REMOVED = "engine-removed"; +const SEARCH_ENGINE_ADDED = "engine-added"; +const SEARCH_ENGINE_CHANGED = "engine-changed"; +const SEARCH_ENGINE_LOADED = "engine-loaded"; +const SEARCH_ENGINE_CURRENT = "engine-current"; +const SEARCH_ENGINE_DEFAULT = "engine-default"; + +// The following constants are left undocumented in nsIBrowserSearchService.idl +// For the moment, they are meant for testing/debugging purposes only. + +/** + * Topic used for events involving the service itself. + */ +const SEARCH_SERVICE_TOPIC = "browser-search-service"; + +/** + * Sent whenever the cache is fully written to disk. + */ +const SEARCH_SERVICE_CACHE_WRITTEN = "write-cache-to-disk-complete"; + +// Delay for lazy serialization (ms) +const LAZY_SERIALIZE_DELAY = 100; + +// Delay for batching invalidation of the JSON cache (ms) +const CACHE_INVALIDATION_DELAY = 1000; + +// Current cache version. This should be incremented if the format of the cache +// file is modified. +const CACHE_VERSION = 1; + +const CACHE_FILENAME = "search.json.mozlz4"; + +const NEW_LINES = /(\r\n|\r|\n)/; + +// Set an arbitrary cap on the maximum icon size. Without this, large icons can +// cause big delays when loading them at startup. +const MAX_ICON_SIZE = 32768; + +// Default charset to use for sending search parameters. ISO-8859-1 is used to +// match previous nsInternetSearchService behavior. +const DEFAULT_QUERY_CHARSET = "ISO-8859-1"; + +const SEARCH_BUNDLE = "chrome://global/locale/search/search.properties"; +const BRAND_BUNDLE = "chrome://branding/locale/brand.properties"; + +const OPENSEARCH_NS_10 = "http://a9.com/-/spec/opensearch/1.0/"; +const OPENSEARCH_NS_11 = "http://a9.com/-/spec/opensearch/1.1/"; + +// Although the specification at http://opensearch.a9.com/spec/1.1/description/ +// gives the namespace names defined above, many existing OpenSearch engines +// are using the following versions. We therefore allow either. +const OPENSEARCH_NAMESPACES = [ + OPENSEARCH_NS_11, OPENSEARCH_NS_10, + "http://a9.com/-/spec/opensearchdescription/1.1/", + "http://a9.com/-/spec/opensearchdescription/1.0/" +]; + +const OPENSEARCH_LOCALNAME = "OpenSearchDescription"; + +const MOZSEARCH_NS_10 = "http://www.mozilla.org/2006/browser/search/"; +const MOZSEARCH_LOCALNAME = "SearchPlugin"; + +const URLTYPE_SUGGEST_JSON = "application/x-suggestions+json"; +const URLTYPE_SEARCH_HTML = "text/html"; +const URLTYPE_OPENSEARCH = "application/opensearchdescription+xml"; + +const BROWSER_SEARCH_PREF = "browser.search."; +const LOCALE_PREF = "general.useragent.locale"; + +const USER_DEFINED = "{searchTerms}"; + +// Custom search parameters +const MOZ_OFFICIAL = AppConstants.MOZ_OFFICIAL_BRANDING ? "official" : "unofficial"; + +const MOZ_PARAM_LOCALE = /\{moz:locale\}/g; +const MOZ_PARAM_DIST_ID = /\{moz:distributionID\}/g; +const MOZ_PARAM_OFFICIAL = /\{moz:official\}/g; + +// Supported OpenSearch parameters +// See http://opensearch.a9.com/spec/1.1/querysyntax/#core +const OS_PARAM_USER_DEFINED = /\{searchTerms\??\}/g; +const OS_PARAM_INPUT_ENCODING = /\{inputEncoding\??\}/g; +const OS_PARAM_LANGUAGE = /\{language\??\}/g; +const OS_PARAM_OUTPUT_ENCODING = /\{outputEncoding\??\}/g; + +// Default values +const OS_PARAM_LANGUAGE_DEF = "*"; +const OS_PARAM_OUTPUT_ENCODING_DEF = "UTF-8"; +const OS_PARAM_INPUT_ENCODING_DEF = "UTF-8"; + +// "Unsupported" OpenSearch parameters. For example, we don't support +// page-based results, so if the engine requires that we send the "page index" +// parameter, we'll always send "1". +const OS_PARAM_COUNT = /\{count\??\}/g; +const OS_PARAM_START_INDEX = /\{startIndex\??\}/g; +const OS_PARAM_START_PAGE = /\{startPage\??\}/g; + +// Default values +const OS_PARAM_COUNT_DEF = "20"; // 20 results +const OS_PARAM_START_INDEX_DEF = "1"; // start at 1st result +const OS_PARAM_START_PAGE_DEF = "1"; // 1st page + +// Optional parameter +const OS_PARAM_OPTIONAL = /\{(?:\w+:)?\w+\?\}/g; + +// A array of arrays containing parameters that we don't fully support, and +// their default values. We will only send values for these parameters if +// required, since our values are just really arbitrary "guesses" that should +// give us the output we want. +var OS_UNSUPPORTED_PARAMS = [ + [OS_PARAM_COUNT, OS_PARAM_COUNT_DEF], + [OS_PARAM_START_INDEX, OS_PARAM_START_INDEX_DEF], + [OS_PARAM_START_PAGE, OS_PARAM_START_PAGE_DEF], +]; + +// The default engine update interval, in days. This is only used if an engine +// specifies an updateURL, but not an updateInterval. +const SEARCH_DEFAULT_UPDATE_INTERVAL = 7; + +// The default interval before checking again for the name of the +// default engine for the region, in seconds. Only used if the response +// from the server doesn't specify an interval. +const SEARCH_GEO_DEFAULT_UPDATE_INTERVAL = 2592000; // 30 days. + +this.__defineGetter__("FileUtils", function() { + delete this.FileUtils; + Components.utils.import("resource://gre/modules/FileUtils.jsm"); + return FileUtils; +}); + +this.__defineGetter__("NetUtil", function() { + delete this.NetUtil; + Components.utils.import("resource://gre/modules/NetUtil.jsm"); + return NetUtil; +}); + +this.__defineGetter__("gChromeReg", function() { + delete this.gChromeReg; + return this.gChromeReg = Cc["@mozilla.org/chrome/chrome-registry;1"]. + getService(Ci.nsIChromeRegistry); +}); + +/** + * Prefixed to all search debug output. + */ +const SEARCH_LOG_PREFIX = "*** Search: "; + +/** + * Outputs aText to the JavaScript console as well as to stdout. + */ +function DO_LOG(aText) { + dump(SEARCH_LOG_PREFIX + aText + "\n"); + Services.console.logStringMessage(aText); +} + +/** + * In debug builds, use a live, pref-based (browser.search.log) LOG function + * to allow enabling/disabling without a restart. Otherwise, don't log at all by + * default. This can be overridden at startup by the pref, see SearchService's + * _init method. + */ +var LOG = function() {}; + +if (AppConstants.DEBUG) { + LOG = function (aText) { + if (getBoolPref(BROWSER_SEARCH_PREF + "log", false)) { + DO_LOG(aText); + } + }; +} + +/** + * Presents an assertion dialog in non-release builds and throws. + * @param message + * A message to display + * @param resultCode + * The NS_ERROR_* value to throw. + * @throws resultCode + */ +function ERROR(message, resultCode) { + NS_ASSERT(false, SEARCH_LOG_PREFIX + message); + throw Components.Exception(message, resultCode); +} + +/** + * Logs the failure message (if browser.search.log is enabled) and throws. + * @param message + * A message to display + * @param resultCode + * The NS_ERROR_* value to throw. + * @throws resultCode or NS_ERROR_INVALID_ARG if resultCode isn't specified. + */ +function FAIL(message, resultCode) { + LOG(message); + throw Components.Exception(message, resultCode || Cr.NS_ERROR_INVALID_ARG); +} + +/** + * Truncates big blobs of (data-)URIs to console-friendly sizes + * @param str + * String to tone down + * @param len + * Maximum length of the string to return. Defaults to the length of a tweet. + */ +function limitURILength(str, len) { + len = len || 140; + if (str.length > len) + return str.slice(0, len) + "..."; + return str; +} + +/** + * Ensures an assertion is met before continuing. Should be used to indicate + * fatal errors. + * @param assertion + * An assertion that must be met + * @param message + * A message to display if the assertion is not met + * @param resultCode + * The NS_ERROR_* value to throw if the assertion is not met + * @throws resultCode + */ +function ENSURE_WARN(assertion, message, resultCode) { + NS_ASSERT(assertion, SEARCH_LOG_PREFIX + message); + if (!assertion) + throw Components.Exception(message, resultCode); +} + +function loadListener(aChannel, aEngine, aCallback) { + this._channel = aChannel; + this._bytes = []; + this._engine = aEngine; + this._callback = aCallback; +} +loadListener.prototype = { + _callback: null, + _channel: null, + _countRead: 0, + _engine: null, + _stream: null, + + QueryInterface: XPCOMUtils.generateQI([ + Ci.nsIRequestObserver, + Ci.nsIStreamListener, + Ci.nsIChannelEventSink, + Ci.nsIInterfaceRequestor, + // See FIXME comment below. + Ci.nsIHttpEventSink, + Ci.nsIProgressEventSink + ]), + + // nsIRequestObserver + onStartRequest: function SRCH_loadStartR(aRequest, aContext) { + LOG("loadListener: Starting request: " + aRequest.name); + this._stream = Cc["@mozilla.org/binaryinputstream;1"]. + createInstance(Ci.nsIBinaryInputStream); + }, + + onStopRequest: function SRCH_loadStopR(aRequest, aContext, aStatusCode) { + LOG("loadListener: Stopping request: " + aRequest.name); + + var requestFailed = !Components.isSuccessCode(aStatusCode); + if (!requestFailed && (aRequest instanceof Ci.nsIHttpChannel)) + requestFailed = !aRequest.requestSucceeded; + + if (requestFailed || this._countRead == 0) { + LOG("loadListener: request failed!"); + // send null so the callback can deal with the failure + this._callback(null, this._engine); + } else + this._callback(this._bytes, this._engine); + this._channel = null; + this._engine = null; + }, + + // nsIStreamListener + onDataAvailable: function SRCH_loadDAvailable(aRequest, aContext, + aInputStream, aOffset, + aCount) { + this._stream.setInputStream(aInputStream); + + // Get a byte array of the data + this._bytes = this._bytes.concat(this._stream.readByteArray(aCount)); + this._countRead += aCount; + }, + + // nsIChannelEventSink + asyncOnChannelRedirect: function SRCH_loadCRedirect(aOldChannel, aNewChannel, + aFlags, callback) { + this._channel = aNewChannel; + callback.onRedirectVerifyCallback(Components.results.NS_OK); + }, + + // nsIInterfaceRequestor + getInterface: function SRCH_load_GI(aIID) { + return this.QueryInterface(aIID); + }, + + // FIXME: bug 253127 + // nsIHttpEventSink + onRedirect: function (aChannel, aNewChannel) {}, + // nsIProgressEventSink + onProgress: function (aRequest, aContext, aProgress, aProgressMax) {}, + onStatus: function (aRequest, aContext, aStatus, aStatusArg) {} +} + +function isPartnerBuild() { + try { + let distroID = Services.prefs.getCharPref("distribution.id"); + + // Mozilla-provided builds (i.e. funnelcake) are not partner builds + if (distroID && !distroID.startsWith("mozilla")) { + return true; + } + } catch (e) {} + + return false; +} + +// Method to determine if we should be using geo-specific defaults +function geoSpecificDefaultsEnabled() { + let geoSpecificDefaults = false; + try { + geoSpecificDefaults = Services.prefs.getBoolPref("browser.search.geoSpecificDefaults"); + } catch (e) {} + + return geoSpecificDefaults; +} + +// Some notes on countryCode and region prefs: +// * A "countryCode" pref is set via a geoip lookup. It always reflects the +// result of that geoip request. +// * A "region" pref, once set, is the region actually used for search. In +// most cases it will be identical to the countryCode pref. +// * The value of "region" and "countryCode" will only not agree in one edge +// case - 34/35 users who have previously been configured to use US defaults +// based purely on a timezone check will have "region" forced to US, +// regardless of what countryCode geoip returns. +// * We may want to know if we are in the US before we have *either* +// countryCode or region - in which case we fallback to a timezone check, +// but we don't persist that value anywhere in the expectation we will +// eventually get a countryCode/region. + +// A method that "migrates" prefs if necessary. +function migrateRegionPrefs() { + // If we already have a "region" pref there's nothing to do. + if (Services.prefs.prefHasUserValue("browser.search.region")) { + return; + } + + // If we have 'isUS' but no 'countryCode' then we are almost certainly + // a profile from Fx 34/35 that set 'isUS' based purely on a timezone + // check. If this said they were US, we force region to be US. + // (But if isUS was false, we leave region alone - we will do a geoip request + // and set the region accordingly) + try { + if (Services.prefs.getBoolPref("browser.search.isUS") && + !Services.prefs.prefHasUserValue("browser.search.countryCode")) { + Services.prefs.setCharPref("browser.search.region", "US"); + } + } catch (ex) { + // no isUS pref, nothing to do. + } + // If we have a countryCode pref but no region pref, just force region + // to be the countryCode. + try { + let countryCode = Services.prefs.getCharPref("browser.search.countryCode"); + if (!Services.prefs.prefHasUserValue("browser.search.region")) { + Services.prefs.setCharPref("browser.search.region", countryCode); + } + } catch (ex) { + // no countryCode pref, nothing to do. + } +} + +// A method to determine if we are in the United States (US) for the search +// service. +// It uses a browser.search.region pref (which typically comes from a geoip +// request) or if that doesn't exist, falls back to a hacky timezone check. +function getIsUS() { + // Regardless of the region or countryCode, non en-US builds are not + // considered to be in the US from the POV of the search service. + if (getLocale() != "en-US") { + return false; + } + + // If we've got a region pref, trust it. + try { + return Services.prefs.getCharPref("browser.search.region") == "US"; + } catch (e) {} + + // So we are en-US but have no region pref - fallback to hacky timezone check. + let isNA = isUSTimezone(); + LOG("getIsUS() fell back to a timezone check with the result=" + isNA); + return isNA; +} + +// Helper method to modify preference keys with geo-specific modifiers, if needed. +function getGeoSpecificPrefName(basepref) { + if (!geoSpecificDefaultsEnabled() || isPartnerBuild()) + return basepref; + if (getIsUS()) + return basepref + ".US"; + return basepref; +} + +// A method that tries to determine if this user is in a US geography. +function isUSTimezone() { + // Timezone assumptions! We assume that if the system clock's timezone is + // between Newfoundland and Hawaii, that the user is in North America. + + // This includes all of South America as well, but we have relatively few + // en-US users there, so that's OK. + + // 150 minutes = 2.5 hours (UTC-2.5), which is + // Newfoundland Daylight Time (http://www.timeanddate.com/time/zones/ndt) + + // 600 minutes = 10 hours (UTC-10), which is + // Hawaii-Aleutian Standard Time (http://www.timeanddate.com/time/zones/hast) + + let UTCOffset = (new Date()).getTimezoneOffset(); + return UTCOffset >= 150 && UTCOffset <= 600; +} + +// A less hacky method that tries to determine our country-code via an XHR +// geoip lookup. +// If this succeeds and we are using an en-US locale, we set the pref used by +// the hacky method above, so isUS() can avoid the hacky timezone method. +// If it fails we don't touch that pref so isUS() does its normal thing. +var ensureKnownCountryCode = Task.async(function* (ss) { + // If we have a country-code already stored in our prefs we trust it. + let countryCode; + try { + countryCode = Services.prefs.getCharPref("browser.search.countryCode"); + } catch (e) {} + + if (!countryCode) { + // We don't have it cached, so fetch it. fetchCountryCode() will call + // storeCountryCode if it gets a result (even if that happens after the + // promise resolves) and fetchRegionDefault. + yield fetchCountryCode(ss); + } else { + // if nothing to do, return early. + if (!geoSpecificDefaultsEnabled()) + return; + + let expir = ss.getGlobalAttr("searchDefaultExpir") || 0; + if (expir > Date.now()) { + // The territory default we have already fetched hasn't expired yet. + // If we have a default engine or a list of visible default engines + // saved, the hashes should be valid, verify them now so that we can + // refetch if they have been tampered with. + let defaultEngine = ss.getVerifiedGlobalAttr("searchDefault"); + let visibleDefaultEngines = ss.getVerifiedGlobalAttr("visibleDefaultEngines"); + if ((defaultEngine || defaultEngine === undefined) && + (visibleDefaultEngines || visibleDefaultEngines === undefined)) { + // No geo defaults, or valid hashes; nothing to do. + return; + } + } + + yield new Promise(resolve => { + let timeoutMS = Services.prefs.getIntPref("browser.search.geoip.timeout"); + let timerId = setTimeout(() => { + timerId = null; + resolve(); + }, timeoutMS); + + let callback = () => { + clearTimeout(timerId); + resolve(); + }; + fetchRegionDefault(ss).then(callback).catch(err => { + Components.utils.reportError(err); + callback(); + }); + }); + } + + // If gInitialized is true then the search service was forced to perform + // a sync initialization during our XHRs - capture this via telemetry. + Services.telemetry.getHistogramById("SEARCH_SERVICE_COUNTRY_FETCH_CAUSED_SYNC_INIT").add(gInitialized); +}); + +// Store the result of the geoip request as well as any other values and +// telemetry which depend on it. +function storeCountryCode(cc) { + // Set the country-code itself. + Services.prefs.setCharPref("browser.search.countryCode", cc); + // And set the region pref if we don't already have a value. + if (!Services.prefs.prefHasUserValue("browser.search.region")) { + Services.prefs.setCharPref("browser.search.region", cc); + } + // and telemetry... + let isTimezoneUS = isUSTimezone(); + if (cc == "US" && !isTimezoneUS) { + Services.telemetry.getHistogramById("SEARCH_SERVICE_US_COUNTRY_MISMATCHED_TIMEZONE").add(1); + } + if (cc != "US" && isTimezoneUS) { + Services.telemetry.getHistogramById("SEARCH_SERVICE_US_TIMEZONE_MISMATCHED_COUNTRY").add(1); + } + // telemetry to compare our geoip response with platform-specific country data. + // On Mac and Windows, we can get a country code via sysinfo + let platformCC = Services.sysinfo.get("countryCode"); + if (platformCC) { + let probeUSMismatched, probeNonUSMismatched; + switch (Services.appinfo.OS) { + case "Darwin": + probeUSMismatched = "SEARCH_SERVICE_US_COUNTRY_MISMATCHED_PLATFORM_OSX"; + probeNonUSMismatched = "SEARCH_SERVICE_NONUS_COUNTRY_MISMATCHED_PLATFORM_OSX"; + break; + case "WINNT": + probeUSMismatched = "SEARCH_SERVICE_US_COUNTRY_MISMATCHED_PLATFORM_WIN"; + probeNonUSMismatched = "SEARCH_SERVICE_NONUS_COUNTRY_MISMATCHED_PLATFORM_WIN"; + break; + default: + Cu.reportError("Platform " + Services.appinfo.OS + " has system country code but no search service telemetry probes"); + break; + } + if (probeUSMismatched && probeNonUSMismatched) { + if (cc == "US" || platformCC == "US") { + // one of the 2 said US, so record if they are the same. + Services.telemetry.getHistogramById(probeUSMismatched).add(cc != platformCC); + } else { + // different country - record if they are the same + Services.telemetry.getHistogramById(probeNonUSMismatched).add(cc != platformCC); + } + } + } +} + +// Get the country we are in via a XHR geoip request. +function fetchCountryCode(ss) { + // values for the SEARCH_SERVICE_COUNTRY_FETCH_RESULT 'enum' telemetry probe. + const TELEMETRY_RESULT_ENUM = { + SUCCESS: 0, + SUCCESS_WITHOUT_DATA: 1, + XHRTIMEOUT: 2, + ERROR: 3, + // Note that we expect to add finer-grained error types here later (eg, + // dns error, network error, ssl error, etc) with .ERROR remaining as the + // generic catch-all that doesn't fit into other categories. + }; + let endpoint = Services.urlFormatter.formatURLPref("browser.search.geoip.url"); + LOG("_fetchCountryCode starting with endpoint " + endpoint); + // As an escape hatch, no endpoint means no geoip. + if (!endpoint) { + return Promise.resolve(); + } + let startTime = Date.now(); + return new Promise(resolve => { + // Instead of using a timeout on the xhr object itself, we simulate one + // using a timer and let the XHR request complete. This allows us to + // capture reliable telemetry on what timeout value should actually be + // used to ensure most users don't see one while not making it so large + // that many users end up doing a sync init of the search service and thus + // would see the jank that implies. + // (Note we do actually use a timeout on the XHR, but that's set to be a + // large value just incase the request never completes - we don't want the + // XHR object to live forever) + let timeoutMS = Services.prefs.getIntPref("browser.search.geoip.timeout"); + let geoipTimeoutPossible = true; + let timerId = setTimeout(() => { + LOG("_fetchCountryCode: timeout fetching country information"); + if (geoipTimeoutPossible) + Services.telemetry.getHistogramById("SEARCH_SERVICE_COUNTRY_TIMEOUT").add(1); + timerId = null; + resolve(); + }, timeoutMS); + + let resolveAndReportSuccess = (result, reason) => { + // Even if we timed out, we want to save the country code and everything + // related so next startup sees the value and doesn't retry this dance. + if (result) { + storeCountryCode(result); + } + Services.telemetry.getHistogramById("SEARCH_SERVICE_COUNTRY_FETCH_RESULT").add(reason); + + // This notification is just for tests... + Services.obs.notifyObservers(null, SEARCH_SERVICE_TOPIC, "geoip-lookup-xhr-complete"); + + if (timerId) { + Services.telemetry.getHistogramById("SEARCH_SERVICE_COUNTRY_TIMEOUT").add(0); + geoipTimeoutPossible = false; + } + + let callback = () => { + // If we've already timed out then we've already resolved the promise, + // so there's nothing else to do. + if (timerId == null) { + return; + } + clearTimeout(timerId); + resolve(); + }; + + if (result && geoSpecificDefaultsEnabled()) { + fetchRegionDefault(ss).then(callback).catch(err => { + Components.utils.reportError(err); + callback(); + }); + } else { + callback(); + } + }; + + let request = new XMLHttpRequest(); + // This notification is just for tests... + Services.obs.notifyObservers(request, SEARCH_SERVICE_TOPIC, "geoip-lookup-xhr-starting"); + request.timeout = 100000; // 100 seconds as the last-chance fallback + request.onload = function(event) { + let took = Date.now() - startTime; + let cc = event.target.response && event.target.response.country_code; + LOG("_fetchCountryCode got success response in " + took + "ms: " + cc); + Services.telemetry.getHistogramById("SEARCH_SERVICE_COUNTRY_FETCH_TIME_MS").add(took); + let reason = cc ? TELEMETRY_RESULT_ENUM.SUCCESS : TELEMETRY_RESULT_ENUM.SUCCESS_WITHOUT_DATA; + resolveAndReportSuccess(cc, reason); + }; + request.ontimeout = function(event) { + LOG("_fetchCountryCode: XHR finally timed-out fetching country information"); + resolveAndReportSuccess(null, TELEMETRY_RESULT_ENUM.XHRTIMEOUT); + }; + request.onerror = function(event) { + LOG("_fetchCountryCode: failed to retrieve country information"); + resolveAndReportSuccess(null, TELEMETRY_RESULT_ENUM.ERROR); + }; + request.open("POST", endpoint, true); + request.setRequestHeader("Content-Type", "application/json"); + request.responseType = "json"; + request.send("{}"); + }); +} + +// This will make an HTTP request to a Mozilla server that will return +// JSON data telling us what engine should be set as the default for +// the current region, and how soon we should check again. +// +// The optional cohort value returned by the server is to be kept locally +// and sent to the server the next time we ping it. It lets the server +// identify profiles that have been part of a specific experiment. +// +// This promise may take up to 100s to resolve, it's the caller's +// responsibility to ensure with a timer that we are not going to +// block the async init for too long. +var fetchRegionDefault = (ss) => new Promise(resolve => { + let urlTemplate = Services.prefs.getDefaultBranch(BROWSER_SEARCH_PREF) + .getCharPref("geoSpecificDefaults.url"); + let endpoint = Services.urlFormatter.formatURL(urlTemplate); + + // As an escape hatch, no endpoint means no region specific defaults. + if (!endpoint) { + resolve(); + return; + } + + // Append the optional cohort value. + const cohortPref = "browser.search.cohort"; + let cohort; + try { + cohort = Services.prefs.getCharPref(cohortPref); + } catch (e) {} + if (cohort) + endpoint += "/" + cohort; + + LOG("fetchRegionDefault starting with endpoint " + endpoint); + + let startTime = Date.now(); + let request = new XMLHttpRequest(); + request.timeout = 100000; // 100 seconds as the last-chance fallback + request.onload = function(event) { + let took = Date.now() - startTime; + + let status = event.target.status; + if (status != 200) { + LOG("fetchRegionDefault failed with HTTP code " + status); + let retryAfter = request.getResponseHeader("retry-after"); + if (retryAfter) { + ss.setGlobalAttr("searchDefaultExpir", Date.now() + retryAfter * 1000); + } + resolve(); + return; + } + + let response = event.target.response || {}; + LOG("received " + response.toSource()); + + if (response.cohort) { + Services.prefs.setCharPref(cohortPref, response.cohort); + } else { + Services.prefs.clearUserPref(cohortPref); + } + + if (response.settings && response.settings.searchDefault) { + let defaultEngine = response.settings.searchDefault; + ss.setVerifiedGlobalAttr("searchDefault", defaultEngine); + LOG("fetchRegionDefault saved searchDefault: " + defaultEngine); + } + + if (response.settings && response.settings.visibleDefaultEngines) { + let visibleDefaultEngines = response.settings.visibleDefaultEngines; + let string = visibleDefaultEngines.join(","); + ss.setVerifiedGlobalAttr("visibleDefaultEngines", string); + LOG("fetchRegionDefault saved visibleDefaultEngines: " + string); + } + + let interval = response.interval || SEARCH_GEO_DEFAULT_UPDATE_INTERVAL; + let milliseconds = interval * 1000; // |interval| is in seconds. + ss.setGlobalAttr("searchDefaultExpir", Date.now() + milliseconds); + + LOG("fetchRegionDefault got success response in " + took + "ms"); + resolve(); + }; + request.ontimeout = function(event) { + LOG("fetchRegionDefault: XHR finally timed-out"); + resolve(); + }; + request.onerror = function(event) { + LOG("fetchRegionDefault: failed to retrieve territory default information"); + resolve(); + }; + request.open("GET", endpoint, true); + request.setRequestHeader("Content-Type", "application/json"); + request.responseType = "json"; + request.send(); +}); + +function getVerificationHash(aName) { + let disclaimer = "By modifying this file, I agree that I am doing so " + + "only within $appName itself, using official, user-driven search " + + "engine selection processes, and in a way which does not circumvent " + + "user consent. I acknowledge that any attempt to change this file " + + "from outside of $appName is a malicious act, and will be responded " + + "to accordingly." + + let salt = OS.Path.basename(OS.Constants.Path.profileDir) + aName + + disclaimer.replace(/\$appName/g, Services.appinfo.name); + + let converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"] + .createInstance(Ci.nsIScriptableUnicodeConverter); + converter.charset = "UTF-8"; + + // Data is an array of bytes. + let data = converter.convertToByteArray(salt, {}); + let hasher = Cc["@mozilla.org/security/hash;1"] + .createInstance(Ci.nsICryptoHash); + hasher.init(hasher.SHA256); + hasher.update(data, data.length); + + return hasher.finish(true); +} + +/** + * Safely close a nsISafeOutputStream. + * @param aFOS + * The file output stream to close. + */ +function closeSafeOutputStream(aFOS) { + if (aFOS instanceof Ci.nsISafeOutputStream) { + try { + aFOS.finish(); + return; + } catch (e) { } + } + aFOS.close(); +} + +/** + * Wrapper function for nsIIOService::newURI. + * @param aURLSpec + * The URL string from which to create an nsIURI. + * @returns an nsIURI object, or null if the creation of the URI failed. + */ +function makeURI(aURLSpec, aCharset) { + try { + return NetUtil.newURI(aURLSpec, aCharset); + } catch (ex) { } + + return null; +} + +/** + * Wrapper function for nsIIOService::newChannel2. + * @param url + * The URL string from which to create an nsIChannel. + * @returns an nsIChannel object, or null if the url is invalid. + */ +function makeChannel(url) { + try { + return NetUtil.newChannel({ + uri: url, + loadUsingSystemPrincipal: true + }); + } catch (ex) { } + + return null; +} + +/** + * Gets a directory from the directory service. + * @param aKey + * The directory service key indicating the directory to get. + */ +function getDir(aKey, aIFace) { + if (!aKey) + FAIL("getDir requires a directory key!"); + + return Services.dirsvc.get(aKey, aIFace || Ci.nsIFile); +} + +/** + * Gets the current value of the locale. It's possible for this preference to + * be localized, so we have to do a little extra work here. Similar code + * exists in nsHttpHandler.cpp when building the UA string. + */ +function getLocale() { + let locale = getLocalizedPref(LOCALE_PREF); + if (locale) + return locale; + + // Not localized. + return Services.prefs.getCharPref(LOCALE_PREF); +} + +/** + * Wrapper for nsIPrefBranch::getComplexValue. + * @param aPrefName + * The name of the pref to get. + * @returns aDefault if the requested pref doesn't exist. + */ +function getLocalizedPref(aPrefName, aDefault) { + const nsIPLS = Ci.nsIPrefLocalizedString; + try { + return Services.prefs.getComplexValue(aPrefName, nsIPLS).data; + } catch (ex) {} + + return aDefault; +} + +/** + * Wrapper for nsIPrefBranch::getBoolPref. + * @param aPrefName + * The name of the pref to get. + * @returns aDefault if the requested pref doesn't exist. + */ +function getBoolPref(aName, aDefault) { + if (Services.prefs.getPrefType(aName) != Ci.nsIPrefBranch.PREF_BOOL) + return aDefault; + return Services.prefs.getBoolPref(aName); +} + +/** + * @return a sanitized name to be used as a filename, or a random name + * if a sanitized name cannot be obtained (if aName contains + * no valid characters). + */ +function sanitizeName(aName) { + const maxLength = 60; + const minLength = 1; + var name = aName.toLowerCase(); + name = name.replace(/\s+/g, "-"); + name = name.replace(/[^-a-z0-9]/g, ""); + + // Use a random name if our input had no valid characters. + if (name.length < minLength) + name = Math.random().toString(36).replace(/^.*\./, ''); + + // Force max length. + return name.substring(0, maxLength); +} + +/** + * Retrieve a pref from the search param branch. + * + * @param prefName + * The name of the pref. + **/ +function getMozParamPref(prefName) { + let branch = Services.prefs.getDefaultBranch(BROWSER_SEARCH_PREF + "param."); + return encodeURIComponent(branch.getCharPref(prefName)); +} + +/** + * Notifies watchers of SEARCH_ENGINE_TOPIC about changes to an engine or to + * the state of the search service. + * + * @param aEngine + * The nsISearchEngine object to which the change applies. + * @param aVerb + * A verb describing the change. + * + * @see nsIBrowserSearchService.idl + */ +var gInitialized = false; +function notifyAction(aEngine, aVerb) { + if (gInitialized) { + LOG("NOTIFY: Engine: \"" + aEngine.name + "\"; Verb: \"" + aVerb + "\""); + Services.obs.notifyObservers(aEngine, SEARCH_ENGINE_TOPIC, aVerb); + } +} + +function parseJsonFromStream(aInputStream) { + const json = Cc["@mozilla.org/dom/json;1"].createInstance(Ci.nsIJSON); + const data = json.decodeFromStream(aInputStream, aInputStream.available()); + return data; +} + +/** + * Simple object representing a name/value pair. + */ +function QueryParameter(aName, aValue, aPurpose) { + if (!aName || (aValue == null)) + FAIL("missing name or value for QueryParameter!"); + + this.name = aName; + this.value = aValue; + this.purpose = aPurpose; +} + +/** + * Perform OpenSearch parameter substitution on aParamValue. + * + * @param aParamValue + * A string containing OpenSearch search parameters. + * @param aSearchTerms + * The user-provided search terms. This string will inserted into + * aParamValue as the value of the OS_PARAM_USER_DEFINED parameter. + * This value must already be escaped appropriately - it is inserted + * as-is. + * @param aEngine + * The engine which owns the string being acted on. + * + * @see http://opensearch.a9.com/spec/1.1/querysyntax/#core + */ +function ParamSubstitution(aParamValue, aSearchTerms, aEngine) { + var value = aParamValue; + + var distributionID = Services.appinfo.distributionID; + try { + distributionID = Services.prefs.getCharPref(BROWSER_SEARCH_PREF + "distributionID"); + } + catch (ex) { } + var official = MOZ_OFFICIAL; + try { + if (Services.prefs.getBoolPref(BROWSER_SEARCH_PREF + "official")) + official = "official"; + else + official = "unofficial"; + } + catch (ex) { } + + // Custom search parameters. These are only available to default search + // engines. + if (aEngine._isDefault) { + value = value.replace(MOZ_PARAM_LOCALE, getLocale()); + value = value.replace(MOZ_PARAM_DIST_ID, distributionID); + value = value.replace(MOZ_PARAM_OFFICIAL, official); + } + + // Insert the OpenSearch parameters we're confident about + value = value.replace(OS_PARAM_USER_DEFINED, aSearchTerms); + value = value.replace(OS_PARAM_INPUT_ENCODING, aEngine.queryCharset); + value = value.replace(OS_PARAM_LANGUAGE, + getLocale() || OS_PARAM_LANGUAGE_DEF); + value = value.replace(OS_PARAM_OUTPUT_ENCODING, + OS_PARAM_OUTPUT_ENCODING_DEF); + + // Replace any optional parameters + value = value.replace(OS_PARAM_OPTIONAL, ""); + + // Insert any remaining required params with our default values + for (var i = 0; i < OS_UNSUPPORTED_PARAMS.length; ++i) { + value = value.replace(OS_UNSUPPORTED_PARAMS[i][0], + OS_UNSUPPORTED_PARAMS[i][1]); + } + + return value; +} + +/** + * Creates an engineURL object, which holds the query URL and all parameters. + * + * @param aType + * A string containing the name of the MIME type of the search results + * returned by this URL. + * @param aMethod + * The HTTP request method. Must be a case insensitive value of either + * "GET" or "POST". + * @param aTemplate + * The URL to which search queries should be sent. For GET requests, + * must contain the string "{searchTerms}", to indicate where the user + * entered search terms should be inserted. + * @param aResultDomain + * The root domain for this URL. Defaults to the template's host. + * + * @see http://opensearch.a9.com/spec/1.1/querysyntax/#urltag + * + * @throws NS_ERROR_NOT_IMPLEMENTED if aType is unsupported. + */ +function EngineURL(aType, aMethod, aTemplate, aResultDomain) { + if (!aType || !aMethod || !aTemplate) + FAIL("missing type, method or template for EngineURL!"); + + var method = aMethod.toUpperCase(); + var type = aType.toLowerCase(); + + if (method != "GET" && method != "POST") + FAIL("method passed to EngineURL must be \"GET\" or \"POST\""); + + this.type = type; + this.method = method; + this.params = []; + this.rels = []; + // Don't serialize expanded mozparams + this.mozparams = {}; + + var templateURI = makeURI(aTemplate); + if (!templateURI) + FAIL("new EngineURL: template is not a valid URI!", Cr.NS_ERROR_FAILURE); + + switch (templateURI.scheme) { + case "http": + case "https": + // Disable these for now, see bug 295018 + // case "file": + // case "resource": + this.template = aTemplate; + break; + default: + FAIL("new EngineURL: template uses invalid scheme!", Cr.NS_ERROR_FAILURE); + } + + // If no resultDomain was specified in the engine definition file, use the + // host from the template. + this.resultDomain = aResultDomain || templateURI.host; + // We never want to return a "www." prefix, so eventually strip it. + if (this.resultDomain.startsWith("www.")) { + this.resultDomain = this.resultDomain.substr(4); + } +} +EngineURL.prototype = { + + addParam: function SRCH_EURL_addParam(aName, aValue, aPurpose) { + this.params.push(new QueryParameter(aName, aValue, aPurpose)); + }, + + // Note: This method requires that aObj has a unique name or the previous MozParams entry with + // that name will be overwritten. + _addMozParam: function SRCH_EURL__addMozParam(aObj) { + aObj.mozparam = true; + this.mozparams[aObj.name] = aObj; + }, + + getSubmission: function SRCH_EURL_getSubmission(aSearchTerms, aEngine, aPurpose) { + var url = ParamSubstitution(this.template, aSearchTerms, aEngine); + // Default to searchbar if the purpose is not provided + var purpose = aPurpose || "searchbar"; + + // If a particular purpose isn't defined in the plugin, fallback to 'searchbar'. + if (!this.params.some(p => p.purpose !== undefined && p.purpose == purpose)) + purpose = "searchbar"; + + // Create an application/x-www-form-urlencoded representation of our params + // (name=value&name=value&name=value) + var dataString = ""; + for (var i = 0; i < this.params.length; ++i) { + var param = this.params[i]; + + // If this parameter has a purpose, only add it if the purpose matches + if (param.purpose !== undefined && param.purpose != purpose) + continue; + + var value = ParamSubstitution(param.value, aSearchTerms, aEngine); + + dataString += (i > 0 ? "&" : "") + param.name + "=" + value; + } + + var postData = null; + if (this.method == "GET") { + // GET method requests have no post data, and append the encoded + // query string to the url... + if (url.indexOf("?") == -1 && dataString) + url += "?"; + url += dataString; + } else if (this.method == "POST") { + // POST method requests must wrap the encoded text in a MIME + // stream and supply that as POSTDATA. + var stringStream = Cc["@mozilla.org/io/string-input-stream;1"]. + createInstance(Ci.nsIStringInputStream); + stringStream.data = dataString; + + postData = Cc["@mozilla.org/network/mime-input-stream;1"]. + createInstance(Ci.nsIMIMEInputStream); + postData.addHeader("Content-Type", "application/x-www-form-urlencoded"); + postData.addContentLength = true; + postData.setData(stringStream); + } + + return new Submission(makeURI(url), postData); + }, + + _getTermsParameterName: function SRCH_EURL__getTermsParameterName() { + let queryParam = this.params.find(p => p.value == USER_DEFINED); + return queryParam ? queryParam.name : ""; + }, + + _hasRelation: function SRC_EURL__hasRelation(aRel) { + return this.rels.some(e => e == aRel.toLowerCase()); + }, + + _initWithJSON: function SRC_EURL__initWithJSON(aJson, aEngine) { + if (!aJson.params) + return; + + this.rels = aJson.rels; + + for (let i = 0; i < aJson.params.length; ++i) { + let param = aJson.params[i]; + if (param.mozparam) { + if (param.condition == "pref") { + let value = getMozParamPref(param.pref); + this.addParam(param.name, value); + } + this._addMozParam(param); + } + else + this.addParam(param.name, param.value, param.purpose || undefined); + } + }, + + /** + * Creates a JavaScript object that represents this URL. + * @returns An object suitable for serialization as JSON. + **/ + toJSON: function SRCH_EURL_toJSON() { + var json = { + template: this.template, + rels: this.rels, + resultDomain: this.resultDomain + }; + + if (this.type != URLTYPE_SEARCH_HTML) + json.type = this.type; + if (this.method != "GET") + json.method = this.method; + + function collapseMozParams(aParam) { + return this.mozparams[aParam.name] || aParam; + } + json.params = this.params.map(collapseMozParams, this); + + return json; + } +}; + +/** + * nsISearchEngine constructor. + * @param aLocation + * A nsILocalFile or nsIURI object representing the location of the + * search engine data file. + * @param aIsReadOnly + * Boolean indicating whether the engine should be treated as read-only. + */ +function Engine(aLocation, aIsReadOnly) { + this._readOnly = aIsReadOnly; + this._urls = []; + this._metaData = {}; + + let file, uri; + if (typeof aLocation == "string") { + this._shortName = aLocation; + } else if (aLocation instanceof Ci.nsILocalFile) { + if (!aIsReadOnly) { + // This is an engine that was installed in NS_APP_USER_SEARCH_DIR by a + // previous version. We are converting the file to an engine stored only + // in JSON, but we need to keep the reference to the profile file to + // remove it if the user ever removes the engine. + this._filePath = aLocation.persistentDescriptor; + } + file = aLocation; + } else if (aLocation instanceof Ci.nsIURI) { + switch (aLocation.scheme) { + case "https": + case "http": + case "ftp": + case "data": + case "file": + case "resource": + case "chrome": + uri = aLocation; + break; + default: + ERROR("Invalid URI passed to the nsISearchEngine constructor", + Cr.NS_ERROR_INVALID_ARG); + } + } else + ERROR("Engine location is neither a File nor a URI object", + Cr.NS_ERROR_INVALID_ARG); + + if (!this._shortName) { + // If we don't have a shortName at this point, it's the first time we load + // this engine, so let's generate the shortName, id and loadPath values. + let shortName; + if (file) { + shortName = file.leafName; + } + else if (uri && uri instanceof Ci.nsIURL) { + if (aIsReadOnly || (gEnvironment.get("XPCSHELL_TEST_PROFILE_DIR") && + uri.scheme == "resource")) { + shortName = uri.fileName; + } + } + if (shortName && shortName.endsWith(".xml")) { + this._shortName = shortName.slice(0, -4); + } + this._loadPath = this.getAnonymizedLoadPath(file, uri); + + if (!shortName && !aIsReadOnly) { + // We are in the process of downloading and installing the engine. + // We'll have the shortName and id once we are done parsing it. + return; + } + + // Build the id used for the legacy metadata storage, so that we + // can do a one-time import of data from old profiles. + if (this._isDefault || + (uri && uri.spec.startsWith(APP_SEARCH_PREFIX))) { + // The second part of the check is to catch engines from language packs. + // They aren't default engines (because they aren't app-shipped), but we + // still need to give their id an [app] prefix for backward compat. + this._id = "[app]/" + this._shortName + ".xml"; + } + else if (!aIsReadOnly) { + this._id = "[profile]/" + this._shortName + ".xml"; + } + else { + // If the engine is neither a default one, nor a user-installed one, + // it must be extension-shipped, so use the full path as id. + LOG("Setting _id to full path for engine from " + this._loadPath); + this._id = file ? file.path : uri.spec; + } + } +} + +Engine.prototype = { + // Data set by the user. + _metaData: null, + // The data describing the engine, in the form of an XML document element. + _data: null, + // Whether or not the engine is readonly. + _readOnly: true, + // Anonymized path of where we initially loaded the engine from. + // This will stay null for engines installed in the profile before we moved + // to a JSON storage. + _loadPath: null, + // The engine's description + _description: "", + // Used to store the engine to replace, if we're an update to an existing + // engine. + _engineToUpdate: null, + // Set to true if the engine has a preferred icon (an icon that should not be + // overridden by a non-preferred icon). + _hasPreferredIcon: null, + // The engine's name. + _name: null, + // The name of the charset used to submit the search terms. + _queryCharset: null, + // The engine's raw SearchForm value (URL string pointing to a search form). + __searchForm: null, + get _searchForm() { + return this.__searchForm; + }, + set _searchForm(aValue) { + if (/^https?:/i.test(aValue)) + this.__searchForm = aValue; + else + LOG("_searchForm: Invalid URL dropped for " + this._name || + "the current engine"); + }, + // Whether to obtain user confirmation before adding the engine. This is only + // used when the engine is first added to the list. + _confirm: false, + // Whether to set this as the current engine as soon as it is loaded. This + // is only used when the engine is first added to the list. + _useNow: false, + // A function to be invoked when this engine object's addition completes (or + // fails). Only used for installation via addEngine. + _installCallback: null, + // The number of days between update checks for new versions + _updateInterval: null, + // The url to check at for a new update + _updateURL: null, + // The url to check for a new icon + _iconUpdateURL: null, + /* The extension ID if added by an extension. */ + _extensionID: null, + + /** + * Retrieves the data from the engine's file. + * The document element is placed in the engine's data field. + */ + _initFromFile: function SRCH_ENG_initFromFile(file) { + if (!file || !file.exists()) + FAIL("File must exist before calling initFromFile!", Cr.NS_ERROR_UNEXPECTED); + + var fileInStream = Cc["@mozilla.org/network/file-input-stream;1"]. + createInstance(Ci.nsIFileInputStream); + + fileInStream.init(file, MODE_RDONLY, FileUtils.PERMS_FILE, false); + + var domParser = Cc["@mozilla.org/xmlextras/domparser;1"]. + createInstance(Ci.nsIDOMParser); + var doc = domParser.parseFromStream(fileInStream, "UTF-8", + file.fileSize, + "text/xml"); + + this._data = doc.documentElement; + fileInStream.close(); + + // Now that the data is loaded, initialize the engine object + this._initFromData(); + }, + + /** + * Retrieves the data from the engine's file asynchronously. + * The document element is placed in the engine's data field. + * + * @param file The file to load the search plugin from. + * + * @returns {Promise} A promise, resolved successfully if initializing from + * data succeeds, rejected if it fails. + */ + _asyncInitFromFile: Task.async(function* (file) { + if (!file || !(yield OS.File.exists(file.path))) + FAIL("File must exist before calling initFromFile!", Cr.NS_ERROR_UNEXPECTED); + + let fileURI = NetUtil.ioService.newFileURI(file); + yield this._retrieveSearchXMLData(fileURI.spec); + + // Now that the data is loaded, initialize the engine object + this._initFromData(); + }), + + /** + * Retrieves the engine data from a URI. Initializes the engine, flushes to + * disk, and notifies the search service once initialization is complete. + * + * @param uri The uri to load the search plugin from. + */ + _initFromURIAndLoad: function SRCH_ENG_initFromURIAndLoad(uri) { + ENSURE_WARN(uri instanceof Ci.nsIURI, + "Must have URI when calling _initFromURIAndLoad!", + Cr.NS_ERROR_UNEXPECTED); + + LOG("_initFromURIAndLoad: Downloading engine from: \"" + uri.spec + "\"."); + + var chan = NetUtil.newChannel({ + uri: uri, + loadUsingSystemPrincipal: true + }); + + if (this._engineToUpdate && (chan instanceof Ci.nsIHttpChannel)) { + var lastModified = this._engineToUpdate.getAttr("updatelastmodified"); + if (lastModified) + chan.setRequestHeader("If-Modified-Since", lastModified, false); + } + this._uri = uri; + var listener = new loadListener(chan, this, this._onLoad); + chan.notificationCallbacks = listener; + chan.asyncOpen2(listener); + }, + + /** + * Retrieves the engine data from a URI asynchronously and initializes it. + * + * @param uri The uri to load the search plugin from. + * + * @returns {Promise} A promise, resolved successfully if retrieveing data + * succeeds. + */ + _asyncInitFromURI: Task.async(function* (uri) { + LOG("_asyncInitFromURI: Loading engine from: \"" + uri.spec + "\"."); + yield this._retrieveSearchXMLData(uri.spec); + // Now that the data is loaded, initialize the engine object + this._initFromData(); + }), + + /** + * Retrieves the engine data for a given URI asynchronously. + * + * @returns {Promise} A promise, resolved successfully if retrieveing data + * succeeds. + */ + _retrieveSearchXMLData: function SRCH_ENG__retrieveSearchXMLData(aURL) { + let deferred = Promise.defer(); + let request = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"]. + createInstance(Ci.nsIXMLHttpRequest); + request.overrideMimeType("text/xml"); + request.onload = (aEvent) => { + let responseXML = aEvent.target.responseXML; + this._data = responseXML.documentElement; + deferred.resolve(); + }; + request.onerror = function(aEvent) { + deferred.resolve(); + }; + request.open("GET", aURL, true); + request.send(); + + return deferred.promise; + }, + + _initFromURISync: function SRCH_ENG_initFromURISync(uri) { + ENSURE_WARN(uri instanceof Ci.nsIURI, + "Must have URI when calling _initFromURISync!", + Cr.NS_ERROR_UNEXPECTED); + + ENSURE_WARN(uri.schemeIs("resource"), "_initFromURISync called for non-resource URI", + Cr.NS_ERROR_FAILURE); + + LOG("_initFromURISync: Loading engine from: \"" + uri.spec + "\"."); + + var chan = NetUtil.newChannel({ + uri: uri, + loadUsingSystemPrincipal: true + }); + + var stream = chan.open2(); + var parser = Cc["@mozilla.org/xmlextras/domparser;1"]. + createInstance(Ci.nsIDOMParser); + var doc = parser.parseFromStream(stream, "UTF-8", stream.available(), "text/xml"); + + this._data = doc.documentElement; + + // Now that the data is loaded, initialize the engine object + this._initFromData(); + }, + + /** + * Attempts to find an EngineURL object in the set of EngineURLs for + * this Engine that has the given type string. (This corresponds to the + * "type" attribute in the "Url" node in the OpenSearch spec.) + * This method will return the first matching URL object found, or null + * if no matching URL is found. + * + * @param aType string to match the EngineURL's type attribute + * @param aRel [optional] only return URLs that with this rel value + */ + _getURLOfType: function SRCH_ENG__getURLOfType(aType, aRel) { + for (var i = 0; i < this._urls.length; ++i) { + if (this._urls[i].type == aType && (!aRel || this._urls[i]._hasRelation(aRel))) + return this._urls[i]; + } + + return null; + }, + + _confirmAddEngine: function SRCH_SVC_confirmAddEngine() { + var stringBundle = Services.strings.createBundle(SEARCH_BUNDLE); + var titleMessage = stringBundle.GetStringFromName("addEngineConfirmTitle"); + + // Display only the hostname portion of the URL. + var dialogMessage = + stringBundle.formatStringFromName("addEngineConfirmation", + [this._name, this._uri.host], 2); + var checkboxMessage = null; + if (!getBoolPref(BROWSER_SEARCH_PREF + "noCurrentEngine", false)) + checkboxMessage = stringBundle.GetStringFromName("addEngineAsCurrentText"); + + var addButtonLabel = + stringBundle.GetStringFromName("addEngineAddButtonLabel"); + + var ps = Services.prompt; + var buttonFlags = (ps.BUTTON_TITLE_IS_STRING * ps.BUTTON_POS_0) + + (ps.BUTTON_TITLE_CANCEL * ps.BUTTON_POS_1) + + ps.BUTTON_POS_0_DEFAULT; + + var checked = {value: false}; + // confirmEx returns the index of the button that was pressed. Since "Add" + // is button 0, we want to return the negation of that value. + var confirm = !ps.confirmEx(null, + titleMessage, + dialogMessage, + buttonFlags, + addButtonLabel, + null, null, // button 1 & 2 names not used + checkboxMessage, + checked); + + return {confirmed: confirm, useNow: checked.value}; + }, + + /** + * Handle the successful download of an engine. Initializes the engine and + * triggers parsing of the data. The engine is then flushed to disk. Notifies + * the search service once initialization is complete. + */ + _onLoad: function SRCH_ENG_onLoad(aBytes, aEngine) { + /** + * Handle an error during the load of an engine by notifying the engine's + * error callback, if any. + */ + function onError(errorCode = Ci.nsISearchInstallCallback.ERROR_UNKNOWN_FAILURE) { + // Notify the callback of the failure + if (aEngine._installCallback) { + aEngine._installCallback(errorCode); + } + } + + function promptError(strings = {}, error = undefined) { + onError(error); + + if (aEngine._engineToUpdate) { + // We're in an update, so just fail quietly + LOG("updating " + aEngine._engineToUpdate.name + " failed"); + return; + } + var brandBundle = Services.strings.createBundle(BRAND_BUNDLE); + var brandName = brandBundle.GetStringFromName("brandShortName"); + + var searchBundle = Services.strings.createBundle(SEARCH_BUNDLE); + var msgStringName = strings.error || "error_loading_engine_msg2"; + var titleStringName = strings.title || "error_loading_engine_title"; + var title = searchBundle.GetStringFromName(titleStringName); + var text = searchBundle.formatStringFromName(msgStringName, + [brandName, aEngine._location], + 2); + + Services.ww.getNewPrompter(null).alert(title, text); + } + + if (!aBytes) { + promptError(); + return; + } + + var parser = Cc["@mozilla.org/xmlextras/domparser;1"]. + createInstance(Ci.nsIDOMParser); + var doc = parser.parseFromBuffer(aBytes, aBytes.length, "text/xml"); + aEngine._data = doc.documentElement; + + try { + // Initialize the engine from the obtained data + aEngine._initFromData(); + } catch (ex) { + LOG("_onLoad: Failed to init engine!\n" + ex); + // Report an error to the user + promptError(); + return; + } + + if (aEngine._engineToUpdate) { + let engineToUpdate = aEngine._engineToUpdate.wrappedJSObject; + + // Make this new engine use the old engine's shortName, and preserve + // metadata. + aEngine._shortName = engineToUpdate._shortName; + Object.keys(engineToUpdate._metaData).forEach(key => { + aEngine.setAttr(key, engineToUpdate.getAttr(key)); + }); + aEngine._loadPath = engineToUpdate._loadPath; + + // Keep track of the last modified date, so that we can make conditional + // requests for future updates. + aEngine.setAttr("updatelastmodified", (new Date()).toUTCString()); + + // Set the new engine's icon, if it doesn't yet have one. + if (!aEngine._iconURI && engineToUpdate._iconURI) + aEngine._iconURI = engineToUpdate._iconURI; + } else { + // Check that when adding a new engine (e.g., not updating an + // existing one), a duplicate engine does not already exist. + if (Services.search.getEngineByName(aEngine.name)) { + // If we're confirming the engine load, then display a "this is a + // duplicate engine" prompt; otherwise, fail silently. + if (aEngine._confirm) { + promptError({ error: "error_duplicate_engine_msg", + title: "error_invalid_engine_title" + }, Ci.nsISearchInstallCallback.ERROR_DUPLICATE_ENGINE); + } else { + onError(Ci.nsISearchInstallCallback.ERROR_DUPLICATE_ENGINE); + } + LOG("_onLoad: duplicate engine found, bailing"); + return; + } + + // If requested, confirm the addition now that we have the title. + // This property is only ever true for engines added via + // nsIBrowserSearchService::addEngine. + if (aEngine._confirm) { + var confirmation = aEngine._confirmAddEngine(); + LOG("_onLoad: confirm is " + confirmation.confirmed + + "; useNow is " + confirmation.useNow); + if (!confirmation.confirmed) { + onError(); + return; + } + aEngine._useNow = confirmation.useNow; + } + + aEngine._shortName = sanitizeName(aEngine.name); + aEngine._loadPath = aEngine.getAnonymizedLoadPath(null, aEngine._uri); + aEngine.setAttr("loadPathHash", getVerificationHash(aEngine._loadPath)); + } + + // Notify the search service of the successful load. It will deal with + // updates by checking aEngine._engineToUpdate. + notifyAction(aEngine, SEARCH_ENGINE_LOADED); + + // Notify the callback if needed + if (aEngine._installCallback) { + aEngine._installCallback(); + } + }, + + /** + * Creates a key by serializing an object that contains the icon's width + * and height. + * + * @param aWidth + * Width of the icon. + * @param aHeight + * Height of the icon. + * @returns key string + */ + _getIconKey: function SRCH_ENG_getIconKey(aWidth, aHeight) { + let keyObj = { + width: aWidth, + height: aHeight + }; + + return JSON.stringify(keyObj); + }, + + /** + * Add an icon to the icon map used by getIconURIBySize() and getIcons(). + * + * @param aWidth + * Width of the icon. + * @param aHeight + * Height of the icon. + * @param aURISpec + * String with the icon's URI. + */ + _addIconToMap: function SRCH_ENG_addIconToMap(aWidth, aHeight, aURISpec) { + if (aWidth == 16 && aHeight == 16) { + // The 16x16 icon is stored in _iconURL, we don't need to store it twice. + return; + } + + // Use an object instead of a Map() because it needs to be serializable. + this._iconMapObj = this._iconMapObj || {}; + let key = this._getIconKey(aWidth, aHeight); + this._iconMapObj[key] = aURISpec; + }, + + /** + * Sets the .iconURI property of the engine. If both aWidth and aHeight are + * provided an entry will be added to _iconMapObj that will enable accessing + * icon's data through getIcons() and getIconURIBySize() APIs. + * + * @param aIconURL + * A URI string pointing to the engine's icon. Must have a http[s], + * ftp, or data scheme. Icons with HTTP[S] or FTP schemes will be + * downloaded and converted to data URIs for storage in the engine + * XML files, if the engine is not readonly. + * @param aIsPreferred + * Whether or not this icon is to be preferred. Preferred icons can + * override non-preferred icons. + * @param aWidth (optional) + * Width of the icon. + * @param aHeight (optional) + * Height of the icon. + */ + _setIcon: function SRCH_ENG_setIcon(aIconURL, aIsPreferred, aWidth, aHeight) { + var uri = makeURI(aIconURL); + + // Ignore bad URIs + if (!uri) + return; + + LOG("_setIcon: Setting icon url \"" + limitURILength(uri.spec) + "\" for engine \"" + + this.name + "\"."); + // Only accept remote icons from http[s] or ftp + switch (uri.scheme) { + case "resource": + case "chrome": + // We only allow chrome and resource icon URLs for built-in search engines + if (!this._isDefault) { + return; + } + // Fall through to the data case + case "data": + if (!this._hasPreferredIcon || aIsPreferred) { + this._iconURI = uri; + notifyAction(this, SEARCH_ENGINE_CHANGED); + this._hasPreferredIcon = aIsPreferred; + } + + if (aWidth && aHeight) { + this._addIconToMap(aWidth, aHeight, aIconURL) + } + break; + case "http": + case "https": + case "ftp": + LOG("_setIcon: Downloading icon: \"" + uri.spec + + "\" for engine: \"" + this.name + "\""); + var chan = NetUtil.newChannel({ + uri: uri, + loadUsingSystemPrincipal: true + }); + + let iconLoadCallback = function (aByteArray, aEngine) { + // This callback may run after we've already set a preferred icon, + // so check again. + if (aEngine._hasPreferredIcon && !aIsPreferred) + return; + + if (!aByteArray || aByteArray.length > MAX_ICON_SIZE) { + LOG("iconLoadCallback: load failed, or the icon was too large!"); + return; + } + + let type = chan.contentType; + if (!type.startsWith("image/")) + type = "image/x-icon"; + let dataURL = "data:" + type + ";base64," + + btoa(String.fromCharCode.apply(null, aByteArray)); + + aEngine._iconURI = makeURI(dataURL); + + if (aWidth && aHeight) { + aEngine._addIconToMap(aWidth, aHeight, dataURL) + } + + notifyAction(aEngine, SEARCH_ENGINE_CHANGED); + aEngine._hasPreferredIcon = aIsPreferred; + }; + + // If we're currently acting as an "update engine", then the callback + // should set the icon on the engine we're updating and not us, since + // |this| might be gone by the time the callback runs. + var engineToSet = this._engineToUpdate || this; + + var listener = new loadListener(chan, engineToSet, iconLoadCallback); + chan.notificationCallbacks = listener; + chan.asyncOpen2(listener); + break; + } + }, + + /** + * Initialize this Engine object from the collected data. + */ + _initFromData: function SRCH_ENG_initFromData() { + ENSURE_WARN(this._data, "Can't init an engine with no data!", + Cr.NS_ERROR_UNEXPECTED); + + // Ensure we have a supported engine type before attempting to parse it. + let element = this._data; + if ((element.localName == MOZSEARCH_LOCALNAME && + element.namespaceURI == MOZSEARCH_NS_10) || + (element.localName == OPENSEARCH_LOCALNAME && + OPENSEARCH_NAMESPACES.indexOf(element.namespaceURI) != -1)) { + LOG("_init: Initing search plugin from " + this._location); + + this._parse(); + + } else + FAIL(this._location + " is not a valid search plugin.", Cr.NS_ERROR_FAILURE); + + // No need to keep a ref to our data (which in some cases can be a document + // element) past this point + this._data = null; + }, + + /** + * Initialize this Engine object from a collection of metadata. + */ + _initFromMetadata: function SRCH_ENG_initMetaData(aName, aIconURL, aAlias, + aDescription, aMethod, + aTemplate, aExtensionID) { + ENSURE_WARN(!this._readOnly, + "Can't call _initFromMetaData on a readonly engine!", + Cr.NS_ERROR_FAILURE); + + this._urls.push(new EngineURL(URLTYPE_SEARCH_HTML, aMethod, aTemplate)); + + this._name = aName; + this.alias = aAlias; + this._description = aDescription; + this._setIcon(aIconURL, true); + this._extensionID = aExtensionID; + }, + + /** + * Extracts data from an OpenSearch URL element and creates an EngineURL + * object which is then added to the engine's list of URLs. + * + * @throws NS_ERROR_FAILURE if a URL object could not be created. + * + * @see http://opensearch.a9.com/spec/1.1/querysyntax/#urltag. + * @see EngineURL() + */ + _parseURL: function SRCH_ENG_parseURL(aElement) { + var type = aElement.getAttribute("type"); + // According to the spec, method is optional, defaulting to "GET" if not + // specified + var method = aElement.getAttribute("method") || "GET"; + var template = aElement.getAttribute("template"); + var resultDomain = aElement.getAttribute("resultdomain"); + + try { + var url = new EngineURL(type, method, template, resultDomain); + } catch (ex) { + FAIL("_parseURL: failed to add " + template + " as a URL", + Cr.NS_ERROR_FAILURE); + } + + if (aElement.hasAttribute("rel")) + url.rels = aElement.getAttribute("rel").toLowerCase().split(/\s+/); + + for (var i = 0; i < aElement.childNodes.length; ++i) { + var param = aElement.childNodes[i]; + if (param.localName == "Param") { + try { + url.addParam(param.getAttribute("name"), param.getAttribute("value")); + } catch (ex) { + // Ignore failure + LOG("_parseURL: Url element has an invalid param"); + } + } else if (param.localName == "MozParam" && + // We only support MozParams for default search engines + this._isDefault) { + var value; + let condition = param.getAttribute("condition"); + + // MozParams must have a condition to be valid + if (!condition) { + let engineLoc = this._location; + let paramName = param.getAttribute("name"); + LOG("_parseURL: MozParam (" + paramName + ") without a condition attribute found parsing engine: " + engineLoc); + continue; + } + + switch (condition) { + case "purpose": + url.addParam(param.getAttribute("name"), + param.getAttribute("value"), + param.getAttribute("purpose")); + // _addMozParam is not needed here since it can be serialized fine without. _addMozParam + // also requires a unique "name" which is not normally the case when @purpose is used. + break; + case "pref": + try { + value = getMozParamPref(param.getAttribute("pref"), value); + url.addParam(param.getAttribute("name"), value); + url._addMozParam({"pref": param.getAttribute("pref"), + "name": param.getAttribute("name"), + "condition": "pref"}); + } catch (e) { } + break; + default: + let engineLoc = this._location; + let paramName = param.getAttribute("name"); + LOG("_parseURL: MozParam (" + paramName + ") has an unknown condition: " + condition + ". Found parsing engine: " + engineLoc); + break; + } + } + } + + this._urls.push(url); + }, + + /** + * Get the icon from an OpenSearch Image element. + * @see http://opensearch.a9.com/spec/1.1/description/#image + */ + _parseImage: function SRCH_ENG_parseImage(aElement) { + LOG("_parseImage: Image textContent: \"" + limitURILength(aElement.textContent) + "\""); + + let width = parseInt(aElement.getAttribute("width"), 10); + let height = parseInt(aElement.getAttribute("height"), 10); + let isPrefered = width == 16 && height == 16; + + if (isNaN(width) || isNaN(height) || width <= 0 || height <=0) { + LOG("OpenSearch image element must have positive width and height."); + return; + } + + this._setIcon(aElement.textContent, isPrefered, width, height); + }, + + /** + * Extract search engine information from the collected data to initialize + * the engine object. + */ + _parse: function SRCH_ENG_parse() { + var doc = this._data; + + // The OpenSearch spec sets a default value for the input encoding. + this._queryCharset = OS_PARAM_INPUT_ENCODING_DEF; + + for (var i = 0; i < doc.childNodes.length; ++i) { + var child = doc.childNodes[i]; + switch (child.localName) { + case "ShortName": + this._name = child.textContent; + break; + case "Description": + this._description = child.textContent; + break; + case "Url": + try { + this._parseURL(child); + } catch (ex) { + // Parsing of the element failed, just skip it. + LOG("_parse: failed to parse URL child: " + ex); + } + break; + case "Image": + this._parseImage(child); + break; + case "InputEncoding": + this._queryCharset = child.textContent.toUpperCase(); + break; + + // Non-OpenSearch elements + case "SearchForm": + this._searchForm = child.textContent; + break; + case "UpdateUrl": + this._updateURL = child.textContent; + break; + case "UpdateInterval": + this._updateInterval = parseInt(child.textContent); + break; + case "IconUpdateUrl": + this._iconUpdateURL = child.textContent; + break; + case "ExtensionID": + this._extensionID = child.textContent; + break; + } + } + if (!this.name || (this._urls.length == 0)) + FAIL("_parse: No name, or missing URL!", Cr.NS_ERROR_FAILURE); + if (!this.supportsResponseType(URLTYPE_SEARCH_HTML)) + FAIL("_parse: No text/html result type!", Cr.NS_ERROR_FAILURE); + }, + + /** + * Init from a JSON record. + **/ + _initWithJSON: function SRCH_ENG__initWithJSON(aJson) { + this._name = aJson._name; + this._shortName = aJson._shortName; + this._loadPath = aJson._loadPath; + this._description = aJson.description; + this._hasPreferredIcon = aJson._hasPreferredIcon == undefined; + this._queryCharset = aJson.queryCharset || DEFAULT_QUERY_CHARSET; + this.__searchForm = aJson.__searchForm; + this._updateInterval = aJson._updateInterval || null; + this._updateURL = aJson._updateURL || null; + this._iconUpdateURL = aJson._iconUpdateURL || null; + this._readOnly = aJson._readOnly == undefined; + this._iconURI = makeURI(aJson._iconURL); + this._iconMapObj = aJson._iconMapObj; + this._metaData = aJson._metaData || {}; + if (aJson.filePath) { + this._filePath = aJson.filePath; + } + if (aJson.dirPath) { + this._dirPath = aJson.dirPath; + this._dirLastModifiedTime = aJson.dirLastModifiedTime; + } + if (aJson.extensionID) { + this._extensionID = aJson.extensionID; + } + for (let i = 0; i < aJson._urls.length; ++i) { + let url = aJson._urls[i]; + let engineURL = new EngineURL(url.type || URLTYPE_SEARCH_HTML, + url.method || "GET", url.template, + url.resultDomain || undefined); + engineURL._initWithJSON(url, this); + this._urls.push(engineURL); + } + }, + + /** + * Creates a JavaScript object that represents this engine. + * @returns An object suitable for serialization as JSON. + **/ + toJSON: function SRCH_ENG_toJSON() { + var json = { + _name: this._name, + _shortName: this._shortName, + _loadPath: this._loadPath, + description: this.description, + __searchForm: this.__searchForm, + _iconURL: this._iconURL, + _iconMapObj: this._iconMapObj, + _metaData: this._metaData, + _urls: this._urls + }; + + if (this._updateInterval) + json._updateInterval = this._updateInterval; + if (this._updateURL) + json._updateURL = this._updateURL; + if (this._iconUpdateURL) + json._iconUpdateURL = this._iconUpdateURL; + if (!this._hasPreferredIcon) + json._hasPreferredIcon = this._hasPreferredIcon; + if (this.queryCharset != DEFAULT_QUERY_CHARSET) + json.queryCharset = this.queryCharset; + if (!this._readOnly) + json._readOnly = this._readOnly; + if (this._filePath) { + // File path is stored so that we can remove legacy xml files + // from the profile if the user removes the engine. + json.filePath = this._filePath; + } + if (this._dirPath) { + // The directory path is only stored for extension-shipped engines, + // it's used to invalidate the cache. + json.dirPath = this._dirPath; + json.dirLastModifiedTime = this._dirLastModifiedTime; + } + if (this._extensionID) { + json.extensionID = this._extensionID; + } + + return json; + }, + + setAttr(name, val) { + this._metaData[name] = val; + }, + + getAttr(name) { + return this._metaData[name] || undefined; + }, + + // nsISearchEngine + get alias() { + return this.getAttr("alias"); + }, + set alias(val) { + var value = val ? val.trim() : null; + this.setAttr("alias", value); + notifyAction(this, SEARCH_ENGINE_CHANGED); + }, + + /** + * Return the built-in identifier of app-provided engines. + * + * Note that this identifier is substantially similar to _id, with the + * following exceptions: + * + * * There is no trailing file extension. + * * There is no [app] prefix. + * + * @return a string identifier, or null. + */ + get identifier() { + // No identifier if If the engine isn't app-provided + return this._isDefault ? this._shortName : null; + }, + + get description() { + return this._description; + }, + + get hidden() { + return this.getAttr("hidden") || false; + }, + set hidden(val) { + var value = !!val; + if (value != this.hidden) { + this.setAttr("hidden", value); + notifyAction(this, SEARCH_ENGINE_CHANGED); + } + }, + + get iconURI() { + if (this._iconURI) + return this._iconURI; + return null; + }, + + get _iconURL() { + if (!this._iconURI) + return ""; + return this._iconURI.spec; + }, + + // Where the engine is being loaded from: will return the URI's spec if the + // engine is being downloaded and does not yet have a file. This is only used + // for logging and error messages. + get _location() { + if (this._uri) + return this._uri.spec; + + return this._loadPath; + }, + + // This indicates where we found the .xml file to load the engine, + // and attempts to hide user-identifiable data (such as username). + getAnonymizedLoadPath(file, uri) { + /* Examples of expected output: + * jar:[app]/omni.ja!browser/engine.xml + * 'browser' here is the name of the chrome package, not a folder. + * [profile]/searchplugins/engine.xml + * [distribution]/searchplugins/common/engine.xml + * [other]/engine.xml + */ + + const NS_XPCOM_CURRENT_PROCESS_DIR = "XCurProcD"; + const NS_APP_USER_PROFILE_50_DIR = "ProfD"; + const XRE_APP_DISTRIBUTION_DIR = "XREAppDist"; + + const knownDirs = { + app: NS_XPCOM_CURRENT_PROCESS_DIR, + profile: NS_APP_USER_PROFILE_50_DIR, + distribution: XRE_APP_DISTRIBUTION_DIR + }; + + let leafName = this._shortName; + if (!leafName) + return "null"; + leafName += ".xml"; + + let prefix = "", suffix = ""; + if (!file) { + if (uri.schemeIs("resource")) { + uri = makeURI(Services.io.getProtocolHandler("resource") + .QueryInterface(Ci.nsISubstitutingProtocolHandler) + .resolveURI(uri)); + } + let scheme = uri.scheme; + let packageName = ""; + if (scheme == "chrome") { + packageName = uri.hostPort; + uri = gChromeReg.convertChromeURL(uri); + } + + if (AppConstants.platform == "android") { + // On Android the omni.ja file isn't at the same path as the binary + // used to start the process. We tweak the path here so that the code + // shared with Desktop will correctly identify files from the omni.ja + // file as coming from the [app] folder. + let appPath = Services.io.getProtocolHandler("resource") + .QueryInterface(Ci.nsIResProtocolHandler) + .getSubstitution("android"); + if (appPath) { + appPath = appPath.spec; + let spec = uri.spec; + if (spec.includes(appPath)) { + let appURI = Services.io.newFileURI(getDir(knownDirs["app"])); + uri = NetUtil.newURI(spec.replace(appPath, appURI.spec)); + } + } + } + + if (uri instanceof Ci.nsINestedURI) { + prefix = "jar:"; + suffix = "!" + packageName + "/" + leafName; + uri = uri.innermostURI; + } + if (uri instanceof Ci.nsIFileURL) { + file = uri.file; + } else { + let path = "[" + scheme + "]"; + if (/^(?:https?|ftp)$/.test(scheme)) { + path += uri.host; + } + return path + "/" + leafName; + } + } + + let id; + let enginePath = file.path; + + for (let key in knownDirs) { + let path; + try { + path = getDir(knownDirs[key]).path; + } catch (e) { + // Getting XRE_APP_DISTRIBUTION_DIR throws during unit tests. + continue; + } + if (enginePath.startsWith(path)) { + id = "[" + key + "]" + enginePath.slice(path.length).replace(/\\/g, "/"); + break; + } + } + + // If the folder doesn't have a known ancestor, don't record its path to + // avoid leaking user identifiable data. + if (!id) + id = "[other]/" + file.leafName; + + return prefix + id + suffix; + }, + + get _isDefault() { + // If we don't have a shortName, the engine is being parsed from a + // downloaded file, so this can't be a default engine. + if (!this._shortName) + return false; + + // An engine is a default one if we initially loaded it from the application + // or distribution directory. + if (/^(?:jar:)?(?:\[app\]|\[distribution\])/.test(this._loadPath)) + return true; + + // If we are using a non-default locale or in the xpcshell test case, + // we'll accept as a 'default' engine anything that has been registered at + // resource://search-plugins/ even if the file doesn't come from the + // application folder. If not, skip costly additional checks. + if (!Services.prefs.prefHasUserValue(LOCALE_PREF) && + !gEnvironment.get("XPCSHELL_TEST_PROFILE_DIR")) + return false; + + // Some xpcshell tests use the search service without registering + // resource://search-plugins/. + if (!Services.io.getProtocolHandler("resource") + .QueryInterface(Ci.nsIResProtocolHandler) + .hasSubstitution("search-plugins")) + return false; + + let uri = makeURI(APP_SEARCH_PREFIX + this._shortName + ".xml"); + if (this.getAnonymizedLoadPath(null, uri) == this._loadPath) { + // This isn't a real default engine, but it's very close. + LOG("_isDefault, pretending " + this._loadPath + " is a default engine"); + return true; + } + + return false; + }, + + get _hasUpdates() { + // Whether or not the engine has an update URL + let selfURL = this._getURLOfType(URLTYPE_OPENSEARCH, "self"); + return !!(this._updateURL || this._iconUpdateURL || selfURL); + }, + + get name() { + return this._name; + }, + + get searchForm() { + return this._getSearchFormWithPurpose(); + }, + + _getSearchFormWithPurpose(aPurpose = "") { + // First look for a <Url rel="searchform"> + var searchFormURL = this._getURLOfType(URLTYPE_SEARCH_HTML, "searchform"); + if (searchFormURL) { + let submission = searchFormURL.getSubmission("", this, aPurpose); + + // If the rel=searchform URL is not type="get" (i.e. has postData), + // ignore it, since we can only return a URL. + if (!submission.postData) + return submission.uri.spec; + } + + if (!this._searchForm) { + // No SearchForm specified in the engine definition file, use the prePath + // (e.g. https://foo.com for https://foo.com/search.php?q=bar). + var htmlUrl = this._getURLOfType(URLTYPE_SEARCH_HTML); + ENSURE_WARN(htmlUrl, "Engine has no HTML URL!", Cr.NS_ERROR_UNEXPECTED); + this._searchForm = makeURI(htmlUrl.template).prePath; + } + + return ParamSubstitution(this._searchForm, "", this); + }, + + get queryCharset() { + if (this._queryCharset) + return this._queryCharset; + return this._queryCharset = "windows-1252"; // the default + }, + + // from nsISearchEngine + addParam: function SRCH_ENG_addParam(aName, aValue, aResponseType) { + if (!aName || (aValue == null)) + FAIL("missing name or value for nsISearchEngine::addParam!"); + ENSURE_WARN(!this._readOnly, + "called nsISearchEngine::addParam on a read-only engine!", + Cr.NS_ERROR_FAILURE); + if (!aResponseType) + aResponseType = URLTYPE_SEARCH_HTML; + + var url = this._getURLOfType(aResponseType); + if (!url) + FAIL("Engine object has no URL for response type " + aResponseType, + Cr.NS_ERROR_FAILURE); + + url.addParam(aName, aValue); + }, + + get _defaultMobileResponseType() { + let type = URLTYPE_SEARCH_HTML; + + let sysInfo = Cc["@mozilla.org/system-info;1"].getService(Ci.nsIPropertyBag2); + let isTablet = sysInfo.get("tablet"); + if (isTablet && this.supportsResponseType("application/x-moz-tabletsearch")) { + // Check for a tablet-specific search URL override + type = "application/x-moz-tabletsearch"; + } else if (!isTablet && this.supportsResponseType("application/x-moz-phonesearch")) { + // Check for a phone-specific search URL override + type = "application/x-moz-phonesearch"; + } + + delete this._defaultMobileResponseType; + return this._defaultMobileResponseType = type; + }, + + get _isWhiteListed() { + let url = this._getURLOfType(URLTYPE_SEARCH_HTML).template; + let hostname = makeURI(url).host; + let whitelist = Services.prefs.getDefaultBranch(BROWSER_SEARCH_PREF) + .getCharPref("reset.whitelist") + .split(","); + if (whitelist.includes(hostname)) { + LOG("The hostname " + hostname + " is white listed, " + + "we won't show the search reset prompt"); + return true; + } + + return false; + }, + + // from nsISearchEngine + getSubmission: function SRCH_ENG_getSubmission(aData, aResponseType, aPurpose) { + if (!aResponseType) { + aResponseType = AppConstants.platform == "android" ? this._defaultMobileResponseType : + URLTYPE_SEARCH_HTML; + } + + if (aResponseType == URLTYPE_SEARCH_HTML && + Services.prefs.getDefaultBranch(BROWSER_SEARCH_PREF).getBoolPref("reset.enabled") && + this.name == Services.search.currentEngine.name && + !this._isDefault && + this.name != Services.search.originalDefaultEngine.name && + (!this.getAttr("loadPathHash") || + this.getAttr("loadPathHash") != getVerificationHash(this._loadPath)) && + !this._isWhiteListed) { + let url = "about:searchreset"; + let data = []; + if (aData) + data.push("data=" + encodeURIComponent(aData)); + if (aPurpose) + data.push("purpose=" + aPurpose); + if (data.length) + url += "?" + data.join("&"); + return new Submission(makeURI(url)); + } + + var url = this._getURLOfType(aResponseType); + + if (!url) + return null; + + if (!aData) { + // Return a dummy submission object with our searchForm attribute + return new Submission(makeURI(this._getSearchFormWithPurpose(aPurpose))); + } + + LOG("getSubmission: In data: \"" + aData + "\"; Purpose: \"" + aPurpose + "\""); + var data = ""; + try { + data = gTextToSubURI.ConvertAndEscape(this.queryCharset, aData); + } catch (ex) { + LOG("getSubmission: Falling back to default queryCharset!"); + data = gTextToSubURI.ConvertAndEscape(DEFAULT_QUERY_CHARSET, aData); + } + LOG("getSubmission: Out data: \"" + data + "\""); + return url.getSubmission(data, this, aPurpose); + }, + + // from nsISearchEngine + supportsResponseType: function SRCH_ENG_supportsResponseType(type) { + return (this._getURLOfType(type) != null); + }, + + // from nsISearchEngine + getResultDomain: function SRCH_ENG_getResultDomain(aResponseType) { + if (!aResponseType) { + aResponseType = AppConstants.platform == "android" ? this._defaultMobileResponseType : + URLTYPE_SEARCH_HTML; + } + + LOG("getResultDomain: responseType: \"" + aResponseType + "\""); + + let url = this._getURLOfType(aResponseType); + if (url) + return url.resultDomain; + return ""; + }, + + /** + * Returns URL parsing properties used by _buildParseSubmissionMap. + */ + getURLParsingInfo: function () { + let responseType = AppConstants.platform == "android" ? this._defaultMobileResponseType : + URLTYPE_SEARCH_HTML; + + LOG("getURLParsingInfo: responseType: \"" + responseType + "\""); + + let url = this._getURLOfType(responseType); + if (!url || url.method != "GET") { + return null; + } + + let termsParameterName = url._getTermsParameterName(); + if (!termsParameterName) { + return null; + } + + let templateUrl = NetUtil.newURI(url.template).QueryInterface(Ci.nsIURL); + return { + mainDomain: templateUrl.host, + path: templateUrl.filePath.toLowerCase(), + termsParameterName: termsParameterName, + }; + }, + + // nsISupports + QueryInterface: XPCOMUtils.generateQI([Ci.nsISearchEngine]), + + get wrappedJSObject() { + return this; + }, + + /** + * Returns a string with the URL to an engine's icon matching both width and + * height. Returns null if icon with specified dimensions is not found. + * + * @param width + * Width of the requested icon. + * @param height + * Height of the requested icon. + */ + getIconURLBySize: function SRCH_ENG_getIconURLBySize(aWidth, aHeight) { + if (aWidth == 16 && aHeight == 16) + return this._iconURL; + + if (!this._iconMapObj) + return null; + + let key = this._getIconKey(aWidth, aHeight); + if (key in this._iconMapObj) { + return this._iconMapObj[key]; + } + return null; + }, + + /** + * Gets an array of all available icons. Each entry is an object with + * width, height and url properties. width and height are numeric and + * represent the icon's dimensions. url is a string with the URL for + * the icon. + */ + getIcons: function SRCH_ENG_getIcons() { + let result = []; + if (this._iconURL) + result.push({width: 16, height: 16, url: this._iconURL}); + + if (!this._iconMapObj) + return result; + + for (let key of Object.keys(this._iconMapObj)) { + let iconSize = JSON.parse(key); + result.push({ + width: iconSize.width, + height: iconSize.height, + url: this._iconMapObj[key] + }); + } + + return result; + }, + + /** + * Opens a speculative connection to the engine's search URI + * (and suggest URI, if different) to reduce request latency + * + * @param options + * An object that must contain the following fields: + * {window} the content window for the window performing the search + * + * @throws NS_ERROR_INVALID_ARG if options is omitted or lacks required + * elemeents + */ + speculativeConnect: function SRCH_ENG_speculativeConnect(options) { + if (!options || !options.window) { + Cu.reportError("invalid options arg passed to nsISearchEngine.speculativeConnect"); + throw Cr.NS_ERROR_INVALID_ARG; + } + let connector = + Services.io.QueryInterface(Components.interfaces.nsISpeculativeConnect); + + let searchURI = this.getSubmission("dummy").uri; + + let callbacks = options.window.QueryInterface(Components.interfaces.nsIInterfaceRequestor) + .getInterface(Components.interfaces.nsIWebNavigation) + .QueryInterface(Components.interfaces.nsILoadContext); + + connector.speculativeConnect(searchURI, callbacks); + + if (this.supportsResponseType(URLTYPE_SUGGEST_JSON)) { + let suggestURI = this.getSubmission("dummy", URLTYPE_SUGGEST_JSON).uri; + if (suggestURI.prePath != searchURI.prePath) + connector.speculativeConnect(suggestURI, callbacks); + } + }, +}; + +// nsISearchSubmission +function Submission(aURI, aPostData = null) { + this._uri = aURI; + this._postData = aPostData; +} +Submission.prototype = { + get uri() { + return this._uri; + }, + get postData() { + return this._postData; + }, + QueryInterface: XPCOMUtils.generateQI([Ci.nsISearchSubmission]) +} + +// nsISearchParseSubmissionResult +function ParseSubmissionResult(aEngine, aTerms, aTermsOffset, aTermsLength) { + this._engine = aEngine; + this._terms = aTerms; + this._termsOffset = aTermsOffset; + this._termsLength = aTermsLength; +} +ParseSubmissionResult.prototype = { + get engine() { + return this._engine; + }, + get terms() { + return this._terms; + }, + get termsOffset() { + return this._termsOffset; + }, + get termsLength() { + return this._termsLength; + }, + QueryInterface: XPCOMUtils.generateQI([Ci.nsISearchParseSubmissionResult]), +} + +const gEmptyParseSubmissionResult = + Object.freeze(new ParseSubmissionResult(null, "", -1, 0)); + +function executeSoon(func) { + Services.tm.mainThread.dispatch(func, Ci.nsIThread.DISPATCH_NORMAL); +} + +/** + * Check for sync initialization has completed or not. + * + * @param {aPromise} A promise. + * + * @returns the value returned by the invoked method. + * @throws NS_ERROR_ALREADY_INITIALIZED if sync initialization has completed. + */ +function checkForSyncCompletion(aPromise) { + return aPromise.then(function(aValue) { + if (gInitialized) { + throw Components.Exception("Synchronous fallback was called and has " + + "finished so no need to pursue asynchronous " + + "initialization", + Cr.NS_ERROR_ALREADY_INITIALIZED); + } + return aValue; + }); +} + +// nsIBrowserSearchService +function SearchService() { + // Replace empty LOG function with the useful one if the log pref is set. + if (getBoolPref(BROWSER_SEARCH_PREF + "log", false)) + LOG = DO_LOG; + + this._initObservers = Promise.defer(); +} + +SearchService.prototype = { + classID: Components.ID("{7319788a-fe93-4db3-9f39-818cf08f4256}"), + + // The current status of initialization. Note that it does not determine if + // initialization is complete, only if an error has been encountered so far. + _initRV: Cr.NS_OK, + + // The boolean indicates that the initialization has started or not. + _initStarted: null, + + // Reading the JSON cache file is the first thing done during initialization. + // During the async init, we save it in a field so that if we have to do a + // sync init before the async init finishes, we can avoid reading the cache + // with sync disk I/O and handling lz4 decompression synchronously. + // This is set back to null as soon as the initialization is finished. + _cacheFileJSON: null, + + // If initialization has not been completed yet, perform synchronous + // initialization. + // Throws in case of initialization error. + _ensureInitialized: function SRCH_SVC__ensureInitialized() { + if (gInitialized) { + if (!Components.isSuccessCode(this._initRV)) { + LOG("_ensureInitialized: failure"); + throw this._initRV; + } + return; + } + + this._syncInit(); + if (!Components.isSuccessCode(this._initRV)) { + throw this._initRV; + } + }, + + // Synchronous implementation of the initializer. + // Used by |_ensureInitialized| as a fallback if initialization is not + // complete. + _syncInit: function SRCH_SVC__syncInit() { + LOG("_syncInit start"); + this._initStarted = true; + migrateRegionPrefs(); + + let cache = this._readCacheFile(); + if (cache.metaData) + this._metaData = cache.metaData; + + try { + this._syncLoadEngines(cache); + } catch (ex) { + this._initRV = Cr.NS_ERROR_FAILURE; + LOG("_syncInit: failure loading engines: " + ex); + } + this._addObservers(); + + gInitialized = true; + this._cacheFileJSON = null; + + this._initObservers.resolve(this._initRV); + + Services.obs.notifyObservers(null, SEARCH_SERVICE_TOPIC, "init-complete"); + Services.telemetry.getHistogramById("SEARCH_SERVICE_INIT_SYNC").add(true); + this._recordEngineTelemetry(); + + LOG("_syncInit end"); + }, + + /** + * Asynchronous implementation of the initializer. + * + * @returns {Promise} A promise, resolved successfully if the initialization + * succeeds. + */ + _asyncInit: Task.async(function* () { + LOG("_asyncInit start"); + + migrateRegionPrefs(); + + // See if we have a cache file so we don't have to parse a bunch of XML. + let cache = {}; + // Not using checkForSyncCompletion here because we want to ensure we + // fetch the country code and geo specific defaults asynchronously even + // if a sync init has been forced. + cache = yield this._asyncReadCacheFile(); + + if (!gInitialized && cache.metaData) + this._metaData = cache.metaData; + + try { + yield checkForSyncCompletion(ensureKnownCountryCode(this)); + } catch (ex) { + if (ex.result == Cr.NS_ERROR_ALREADY_INITIALIZED) { + throw ex; + } + LOG("_asyncInit: failure determining country code: " + ex); + } + try { + yield checkForSyncCompletion(this._asyncLoadEngines(cache)); + } catch (ex) { + if (ex.result == Cr.NS_ERROR_ALREADY_INITIALIZED) { + throw ex; + } + this._initRV = Cr.NS_ERROR_FAILURE; + LOG("_asyncInit: failure loading engines: " + ex); + } + this._addObservers(); + gInitialized = true; + this._cacheFileJSON = null; + this._initObservers.resolve(this._initRV); + Services.obs.notifyObservers(null, SEARCH_SERVICE_TOPIC, "init-complete"); + Services.telemetry.getHistogramById("SEARCH_SERVICE_INIT_SYNC").add(false); + this._recordEngineTelemetry(); + + LOG("_asyncInit: Completed _asyncInit"); + }), + + _metaData: { }, + setGlobalAttr(name, val) { + this._metaData[name] = val; + this.batchTask.disarm(); + this.batchTask.arm(); + }, + setVerifiedGlobalAttr(name, val) { + this.setGlobalAttr(name, val); + this.setGlobalAttr(name + "Hash", getVerificationHash(val)); + }, + + getGlobalAttr(name) { + return this._metaData[name] || undefined; + }, + getVerifiedGlobalAttr(name) { + let val = this.getGlobalAttr(name); + if (val && this.getGlobalAttr(name + "Hash") != getVerificationHash(val)) { + LOG("getVerifiedGlobalAttr, invalid hash for " + name); + return ""; + } + return val; + }, + + _engines: { }, + __sortedEngines: null, + _visibleDefaultEngines: [], + get _sortedEngines() { + if (!this.__sortedEngines) + return this._buildSortedEngineList(); + return this.__sortedEngines; + }, + + // Get the original Engine object that is the default for this region, + // ignoring changes the user may have subsequently made. + get originalDefaultEngine() { + let defaultEngine = this.getVerifiedGlobalAttr("searchDefault"); + if (!defaultEngine) { + let defaultPrefB = Services.prefs.getDefaultBranch(BROWSER_SEARCH_PREF); + let nsIPLS = Ci.nsIPrefLocalizedString; + + let defPref = getGeoSpecificPrefName("defaultenginename"); + try { + defaultEngine = defaultPrefB.getComplexValue(defPref, nsIPLS).data; + } catch (ex) { + // If the default pref is invalid (e.g. an add-on set it to a bogus value) + // getEngineByName will just return null, which is the best we can do. + } + } + + return this.getEngineByName(defaultEngine); + }, + + resetToOriginalDefaultEngine: function SRCH_SVC__resetToOriginalDefaultEngine() { + this.currentEngine = this.originalDefaultEngine; + }, + + _buildCache: function SRCH_SVC__buildCache() { + if (this._batchTask) + this._batchTask.disarm(); + + let cache = {}; + let locale = getLocale(); + let buildID = Services.appinfo.platformBuildID; + + // Allows us to force a cache refresh should the cache format change. + cache.version = CACHE_VERSION; + // We don't want to incur the costs of stat()ing each plugin on every + // startup when the only (supported) time they will change is during + // app updates (where the buildID is obviously going to change). + // Extension-shipped plugins are the only exception to this, but their + // directories are blown away during updates, so we'll detect their changes. + cache.buildID = buildID; + cache.locale = locale; + + cache.visibleDefaultEngines = this._visibleDefaultEngines; + cache.metaData = this._metaData; + cache.engines = []; + + for (let name in this._engines) { + cache.engines.push(this._engines[name]); + } + + try { + if (!cache.engines.length) + throw "cannot write without any engine."; + + LOG("_buildCache: Writing to cache file."); + let path = OS.Path.join(OS.Constants.Path.profileDir, CACHE_FILENAME); + let data = gEncoder.encode(JSON.stringify(cache)); + let promise = OS.File.writeAtomic(path, data, {compression: "lz4", + tmpPath: path + ".tmp"}); + + promise.then( + function onSuccess() { + Services.obs.notifyObservers(null, SEARCH_SERVICE_TOPIC, SEARCH_SERVICE_CACHE_WRITTEN); + }, + function onError(e) { + LOG("_buildCache: failure during writeAtomic: " + e); + } + ); + } catch (ex) { + LOG("_buildCache: Could not write to cache file: " + ex); + } + }, + + _syncLoadEngines: function SRCH_SVC__syncLoadEngines(cache) { + LOG("_syncLoadEngines: start"); + // See if we have a cache file so we don't have to parse a bunch of XML. + let chromeURIs = this._findJAREngines(); + + let distDirs = []; + let locations; + try { + locations = getDir(NS_APP_DISTRIBUTION_SEARCH_DIR_LIST, + Ci.nsISimpleEnumerator); + } catch (e) { + // NS_APP_DISTRIBUTION_SEARCH_DIR_LIST is defined by each app + // so this throws during unit tests (but not xpcshell tests). + locations = {hasMoreElements: () => false}; + } + while (locations.hasMoreElements()) { + let dir = locations.getNext().QueryInterface(Ci.nsIFile); + if (dir.directoryEntries.hasMoreElements()) + distDirs.push(dir); + } + + let otherDirs = []; + let userSearchDir = getDir(NS_APP_USER_SEARCH_DIR); + locations = getDir(NS_APP_SEARCH_DIR_LIST, Ci.nsISimpleEnumerator); + while (locations.hasMoreElements()) { + let dir = locations.getNext().QueryInterface(Ci.nsIFile); + if ((!cache.engines || !dir.equals(userSearchDir)) && + dir.directoryEntries.hasMoreElements()) + otherDirs.push(dir); + } + + function modifiedDir(aDir) { + return cacheOtherPaths.get(aDir.path) != aDir.lastModifiedTime; + } + + function notInCacheVisibleEngines(aEngineName) { + return cache.visibleDefaultEngines.indexOf(aEngineName) == -1; + } + + let buildID = Services.appinfo.platformBuildID; + let cacheOtherPaths = new Map(); + if (cache.engines) { + for (let engine of cache.engines) { + if (engine._dirPath) { + cacheOtherPaths.set(engine._dirPath, engine._dirLastModifiedTime); + } + } + } + + let rebuildCache = !cache.engines || + cache.version != CACHE_VERSION || + cache.locale != getLocale() || + cache.buildID != buildID || + cacheOtherPaths.size != otherDirs.length || + otherDirs.some(d => !cacheOtherPaths.has(d.path)) || + cache.visibleDefaultEngines.length != this._visibleDefaultEngines.length || + this._visibleDefaultEngines.some(notInCacheVisibleEngines) || + otherDirs.some(modifiedDir); + + if (rebuildCache) { + LOG("_loadEngines: Absent or outdated cache. Loading engines from disk."); + distDirs.forEach(this._loadEnginesFromDir, this); + + this._loadFromChromeURLs(chromeURIs); + + LOG("_loadEngines: load user-installed engines from the obsolete cache"); + this._loadEnginesFromCache(cache, true); + + otherDirs.forEach(this._loadEnginesFromDir, this); + + this._loadEnginesMetadataFromCache(cache); + this._buildCache(); + return; + } + + LOG("_loadEngines: loading from cache directories"); + this._loadEnginesFromCache(cache); + + LOG("_loadEngines: done"); + }, + + /** + * Loads engines asynchronously. + * + * @returns {Promise} A promise, resolved successfully if loading data + * succeeds. + */ + _asyncLoadEngines: Task.async(function* (cache) { + LOG("_asyncLoadEngines: start"); + Services.obs.notifyObservers(null, SEARCH_SERVICE_TOPIC, "find-jar-engines"); + let chromeURIs = + yield checkForSyncCompletion(this._asyncFindJAREngines()); + + // Get the non-empty distribution directories into distDirs... + let distDirs = []; + let locations; + try { + locations = getDir(NS_APP_DISTRIBUTION_SEARCH_DIR_LIST, + Ci.nsISimpleEnumerator); + } catch (e) { + // NS_APP_DISTRIBUTION_SEARCH_DIR_LIST is defined by each app + // so this throws during unit tests (but not xpcshell tests). + locations = {hasMoreElements: () => false}; + } + while (locations.hasMoreElements()) { + let dir = locations.getNext().QueryInterface(Ci.nsIFile); + let iterator = new OS.File.DirectoryIterator(dir.path, + { winPattern: "*.xml" }); + try { + // Add dir to distDirs if it contains any files. + yield checkForSyncCompletion(iterator.next()); + distDirs.push(dir); + } catch (ex) { + // Catch for StopIteration exception. + if (ex.result == Cr.NS_ERROR_ALREADY_INITIALIZED) { + throw ex; + } + } finally { + iterator.close(); + } + } + + // Add the non-empty directories of NS_APP_SEARCH_DIR_LIST to + // otherDirs... + let otherDirs = []; + let userSearchDir = getDir(NS_APP_USER_SEARCH_DIR); + locations = getDir(NS_APP_SEARCH_DIR_LIST, Ci.nsISimpleEnumerator); + while (locations.hasMoreElements()) { + let dir = locations.getNext().QueryInterface(Ci.nsIFile); + if (cache.engines && dir.equals(userSearchDir)) + continue; + let iterator = new OS.File.DirectoryIterator(dir.path, + { winPattern: "*.xml" }); + try { + // Add dir to otherDirs if it contains any files. + yield checkForSyncCompletion(iterator.next()); + otherDirs.push(dir); + } catch (ex) { + // Catch for StopIteration exception. + if (ex.result == Cr.NS_ERROR_ALREADY_INITIALIZED) { + throw ex; + } + } finally { + iterator.close(); + } + } + + let hasModifiedDir = Task.async(function* (aList) { + let modifiedDir = false; + + for (let dir of aList) { + let lastModifiedTime = cacheOtherPaths.get(dir.path); + if (!lastModifiedTime) { + continue; + } + + let info = yield OS.File.stat(dir.path); + if (lastModifiedTime != info.lastModificationDate.getTime()) { + modifiedDir = true; + break; + } + } + return modifiedDir; + }); + + function notInCacheVisibleEngines(aEngineName) { + return cache.visibleDefaultEngines.indexOf(aEngineName) == -1; + } + + let buildID = Services.appinfo.platformBuildID; + let cacheOtherPaths = new Map(); + if (cache.engines) { + for (let engine of cache.engines) { + if (engine._dirPath) { + cacheOtherPaths.set(engine._dirPath, engine._dirLastModifiedTime); + } + } + } + + let rebuildCache = !cache.engines || + cache.version != CACHE_VERSION || + cache.locale != getLocale() || + cache.buildID != buildID || + cacheOtherPaths.size != otherDirs.length || + otherDirs.some(d => !cacheOtherPaths.has(d.path)) || + cache.visibleDefaultEngines.length != this._visibleDefaultEngines.length || + this._visibleDefaultEngines.some(notInCacheVisibleEngines) || + (yield checkForSyncCompletion(hasModifiedDir(otherDirs))); + + if (rebuildCache) { + LOG("_asyncLoadEngines: Absent or outdated cache. Loading engines from disk."); + for (let loadDir of distDirs) { + let enginesFromDir = + yield checkForSyncCompletion(this._asyncLoadEnginesFromDir(loadDir)); + enginesFromDir.forEach(this._addEngineToStore, this); + } + let enginesFromURLs = + yield checkForSyncCompletion(this._asyncLoadFromChromeURLs(chromeURIs)); + enginesFromURLs.forEach(this._addEngineToStore, this); + + LOG("_asyncLoadEngines: loading user-installed engines from the obsolete cache"); + this._loadEnginesFromCache(cache, true); + + for (let loadDir of otherDirs) { + let enginesFromDir = + yield checkForSyncCompletion(this._asyncLoadEnginesFromDir(loadDir)); + enginesFromDir.forEach(this._addEngineToStore, this); + } + + this._loadEnginesMetadataFromCache(cache); + this._buildCache(); + return; + } + + LOG("_asyncLoadEngines: loading from cache directories"); + this._loadEnginesFromCache(cache); + + LOG("_asyncLoadEngines: done"); + }), + + _asyncReInit: function () { + LOG("_asyncReInit"); + // Start by clearing the initialized state, so we don't abort early. + gInitialized = false; + + Task.spawn(function* () { + try { + if (this._batchTask) { + LOG("finalizing batch task"); + let task = this._batchTask; + this._batchTask = null; + yield task.finalize(); + } + + // Clear the engines, too, so we don't stick with the stale ones. + this._engines = {}; + this.__sortedEngines = null; + this._currentEngine = null; + this._visibleDefaultEngines = []; + this._metaData = {}; + this._cacheFileJSON = null; + + // Tests that want to force a synchronous re-initialization need to + // be notified when we are done uninitializing. + Services.obs.notifyObservers(null, SEARCH_SERVICE_TOPIC, + "uninit-complete"); + + let cache = {}; + cache = yield this._asyncReadCacheFile(); + if (!gInitialized && cache.metaData) + this._metaData = cache.metaData; + + yield ensureKnownCountryCode(this); + // Due to the HTTP requests done by ensureKnownCountryCode, it's possible that + // at this point a synchronous init has been forced by other code. + if (!gInitialized) + yield this._asyncLoadEngines(cache); + + // Typically we'll re-init as a result of a pref observer, + // so signal to 'callers' that we're done. + Services.obs.notifyObservers(null, SEARCH_SERVICE_TOPIC, "init-complete"); + this._recordEngineTelemetry(); + gInitialized = true; + } catch (err) { + LOG("Reinit failed: " + err); + Services.obs.notifyObservers(null, SEARCH_SERVICE_TOPIC, "reinit-failed"); + } finally { + Services.obs.notifyObservers(null, SEARCH_SERVICE_TOPIC, "reinit-complete"); + } + }.bind(this)); + }, + + /** + * Read the cache file synchronously. This also imports data from the old + * search-metadata.json file if needed. + * + * @returns A JS object containing the cached data. + */ + _readCacheFile: function SRCH_SVC__readCacheFile() { + if (this._cacheFileJSON) { + return this._cacheFileJSON; + } + + let cacheFile = getDir(NS_APP_USER_PROFILE_50_DIR); + cacheFile.append(CACHE_FILENAME); + + let stream; + try { + stream = Cc["@mozilla.org/network/file-input-stream;1"]. + createInstance(Ci.nsIFileInputStream); + stream.init(cacheFile, MODE_RDONLY, FileUtils.PERMS_FILE, 0); + + let bis = Cc["@mozilla.org/binaryinputstream;1"] + .createInstance(Ci.nsIBinaryInputStream); + bis.setInputStream(stream); + + let count = stream.available(); + let array = new Uint8Array(count); + bis.readArrayBuffer(count, array.buffer); + + let bytes = Lz4.decompressFileContent(array); + let json = JSON.parse(new TextDecoder().decode(bytes)); + if (!json.engines || !json.engines.length) + throw "no engine in the file"; + return json; + } catch (ex) { + LOG("_readCacheFile: Error reading cache file: " + ex); + } finally { + stream.close(); + } + + try { + cacheFile.leafName = "search-metadata.json"; + stream = Cc["@mozilla.org/network/file-input-stream;1"]. + createInstance(Ci.nsIFileInputStream); + stream.init(cacheFile, MODE_RDONLY, FileUtils.PERMS_FILE, 0); + let metadata = parseJsonFromStream(stream); + let json = {}; + if ("[global]" in metadata) { + LOG("_readCacheFile: migrating metadata from search-metadata.json"); + let data = metadata["[global]"]; + json.metaData = {}; + let fields = ["searchDefault", "searchDefaultHash", "searchDefaultExpir", + "current", "hash", + "visibleDefaultEngines", "visibleDefaultEnginesHash"]; + for (let field of fields) { + let name = field.toLowerCase(); + if (name in data) + json.metaData[field] = data[name]; + } + } + delete metadata["[global]"]; + json._oldMetadata = metadata; + + return json; + } catch (ex) { + LOG("_readCacheFile: failed to read old metadata: " + ex); + return {}; + } finally { + stream.close(); + } + }, + + /** + * Read the cache file asynchronously. This also imports data from the old + * search-metadata.json file if needed. + * + * @returns {Promise} A promise, resolved successfully if retrieveing data + * succeeds. + */ + _asyncReadCacheFile: Task.async(function* () { + let json; + try { + let cacheFilePath = OS.Path.join(OS.Constants.Path.profileDir, CACHE_FILENAME); + let bytes = yield OS.File.read(cacheFilePath, {compression: "lz4"}); + json = JSON.parse(new TextDecoder().decode(bytes)); + if (!json.engines || !json.engines.length) + throw "no engine in the file"; + this._cacheFileJSON = json; + } catch (ex) { + LOG("_asyncReadCacheFile: Error reading cache file: " + ex); + json = {}; + + let oldMetadata = + OS.Path.join(OS.Constants.Path.profileDir, "search-metadata.json"); + try { + let bytes = yield OS.File.read(oldMetadata); + let metadata = JSON.parse(new TextDecoder().decode(bytes)); + if ("[global]" in metadata) { + LOG("_asyncReadCacheFile: migrating metadata from search-metadata.json"); + let data = metadata["[global]"]; + json.metaData = {}; + let fields = ["searchDefault", "searchDefaultHash", "searchDefaultExpir", + "current", "hash", + "visibleDefaultEngines", "visibleDefaultEnginesHash"]; + for (let field of fields) { + let name = field.toLowerCase(); + if (name in data) + json.metaData[field] = data[name]; + } + } + delete metadata["[global]"]; + json._oldMetadata = metadata; + } catch (ex) {} + } + return json; + }), + + _batchTask: null, + get batchTask() { + if (!this._batchTask) { + let task = function taskCallback() { + LOG("batchTask: Invalidating engine cache"); + this._buildCache(); + }.bind(this); + this._batchTask = new DeferredTask(task, CACHE_INVALIDATION_DELAY); + } + return this._batchTask; + }, + + _addEngineToStore: function SRCH_SVC_addEngineToStore(aEngine) { + LOG("_addEngineToStore: Adding engine: \"" + aEngine.name + "\""); + + // See if there is an existing engine with the same name. However, if this + // engine is updating another engine, it's allowed to have the same name. + var hasSameNameAsUpdate = (aEngine._engineToUpdate && + aEngine.name == aEngine._engineToUpdate.name); + if (aEngine.name in this._engines && !hasSameNameAsUpdate) { + LOG("_addEngineToStore: Duplicate engine found, aborting!"); + return; + } + + if (aEngine._engineToUpdate) { + // We need to replace engineToUpdate with the engine that just loaded. + var oldEngine = aEngine._engineToUpdate; + + // Remove the old engine from the hash, since it's keyed by name, and our + // name might change (the update might have a new name). + delete this._engines[oldEngine.name]; + + // Hack: we want to replace the old engine with the new one, but since + // people may be holding refs to the nsISearchEngine objects themselves, + // we'll just copy over all "private" properties (those without a getter + // or setter) from one object to the other. + for (var p in aEngine) { + if (!(aEngine.__lookupGetter__(p) || aEngine.__lookupSetter__(p))) + oldEngine[p] = aEngine[p]; + } + aEngine = oldEngine; + aEngine._engineToUpdate = null; + + // Add the engine back + this._engines[aEngine.name] = aEngine; + notifyAction(aEngine, SEARCH_ENGINE_CHANGED); + } else { + // Not an update, just add the new engine. + this._engines[aEngine.name] = aEngine; + // Only add the engine to the list of sorted engines if the initial list + // has already been built (i.e. if this.__sortedEngines is non-null). If + // it hasn't, we're loading engines from disk and the sorted engine list + // will be built once we need it. + if (this.__sortedEngines) { + this.__sortedEngines.push(aEngine); + this._saveSortedEngineList(); + } + notifyAction(aEngine, SEARCH_ENGINE_ADDED); + } + + if (aEngine._hasUpdates) { + // Schedule the engine's next update, if it isn't already. + if (!aEngine.getAttr("updateexpir")) + engineUpdateService.scheduleNextUpdate(aEngine); + } + }, + + _loadEnginesMetadataFromCache: function SRCH_SVC__loadEnginesMetadataFromCache(cache) { + if (cache._oldMetadata) { + // If we have old metadata in the cache, we had no valid cache + // file and read data from search-metadata.json. + for (let name in this._engines) { + let engine = this._engines[name]; + if (engine._id && cache._oldMetadata[engine._id]) + engine._metaData = cache._oldMetadata[engine._id]; + } + return; + } + + if (!cache.engines) + return; + + for (let engine of cache.engines) { + let name = engine._name; + if (name in this._engines) { + LOG("_loadEnginesMetadataFromCache, transfering metadata for " + name); + this._engines[name]._metaData = engine._metaData; + } + } + }, + + _loadEnginesFromCache: function SRCH_SVC__loadEnginesFromCache(cache, + skipReadOnly) { + if (!cache.engines) + return; + + LOG("_loadEnginesFromCache: Loading " + + cache.engines.length + " engines from cache"); + + let skippedEngines = 0; + for (let engine of cache.engines) { + if (skipReadOnly && engine._readOnly == undefined) { + ++skippedEngines; + continue; + } + + this._loadEngineFromCache(engine); + } + + if (skippedEngines) { + LOG("_loadEnginesFromCache: skipped " + skippedEngines + " read-only engines."); + } + }, + + _loadEngineFromCache: function SRCH_SVC__loadEngineFromCache(json) { + try { + let engine = new Engine(json._shortName, json._readOnly == undefined); + engine._initWithJSON(json); + this._addEngineToStore(engine); + } catch (ex) { + LOG("Failed to load " + json._name + " from cache: " + ex); + LOG("Engine JSON: " + json.toSource()); + } + }, + + _loadEnginesFromDir: function SRCH_SVC__loadEnginesFromDir(aDir) { + LOG("_loadEnginesFromDir: Searching in " + aDir.path + " for search engines."); + + // Check whether aDir is the user profile dir + var isInProfile = aDir.equals(getDir(NS_APP_USER_SEARCH_DIR)); + + var files = aDir.directoryEntries + .QueryInterface(Ci.nsIDirectoryEnumerator); + + while (files.hasMoreElements()) { + var file = files.nextFile; + + // Ignore hidden and empty files, and directories + if (!file.isFile() || file.fileSize == 0 || file.isHidden()) + continue; + + var fileURL = NetUtil.ioService.newFileURI(file).QueryInterface(Ci.nsIURL); + var fileExtension = fileURL.fileExtension.toLowerCase(); + + if (fileExtension != "xml") { + // Not an engine + continue; + } + + var addedEngine = null; + try { + addedEngine = new Engine(file, !isInProfile); + addedEngine._initFromFile(file); + if (!isInProfile && !addedEngine._isDefault) { + addedEngine._dirPath = aDir.path; + addedEngine._dirLastModifiedTime = aDir.lastModifiedTime; + } + } catch (ex) { + LOG("_loadEnginesFromDir: Failed to load " + file.path + "!\n" + ex); + continue; + } + + this._addEngineToStore(addedEngine); + } + }, + + /** + * Loads engines from a given directory asynchronously. + * + * @param aDir the directory. + * + * @returns {Promise} A promise, resolved successfully if retrieveing data + * succeeds. + */ + _asyncLoadEnginesFromDir: Task.async(function* (aDir) { + LOG("_asyncLoadEnginesFromDir: Searching in " + aDir.path + " for search engines."); + + // Check whether aDir is the user profile dir + let isInProfile = aDir.equals(getDir(NS_APP_USER_SEARCH_DIR)); + let dirPath = aDir.path; + let iterator = new OS.File.DirectoryIterator(dirPath); + + let osfiles = yield iterator.nextBatch(); + iterator.close(); + + let engines = []; + for (let osfile of osfiles) { + if (osfile.isDir || osfile.isSymLink) + continue; + + let fileInfo = yield OS.File.stat(osfile.path); + if (fileInfo.size == 0) + continue; + + let parts = osfile.path.split("."); + if (parts.length <= 1 || (parts.pop()).toLowerCase() != "xml") { + // Not an engine + continue; + } + + let addedEngine = null; + try { + let file = new FileUtils.File(osfile.path); + addedEngine = new Engine(file, !isInProfile); + yield checkForSyncCompletion(addedEngine._asyncInitFromFile(file)); + if (!isInProfile && !addedEngine._isDefault) { + addedEngine._dirPath = dirPath; + let info = yield OS.File.stat(dirPath); + addedEngine._dirLastModifiedTime = + info.lastModificationDate.getTime(); + } + engines.push(addedEngine); + } catch (ex) { + if (ex.result == Cr.NS_ERROR_ALREADY_INITIALIZED) { + throw ex; + } + LOG("_asyncLoadEnginesFromDir: Failed to load " + osfile.path + "!\n" + ex); + } + } + return engines; + }), + + _loadFromChromeURLs: function SRCH_SVC_loadFromChromeURLs(aURLs) { + aURLs.forEach(function (url) { + try { + LOG("_loadFromChromeURLs: loading engine from chrome url: " + url); + + let uri = makeURI(url); + let engine = new Engine(uri, true); + + engine._initFromURISync(uri); + + this._addEngineToStore(engine); + } catch (ex) { + LOG("_loadFromChromeURLs: failed to load engine: " + ex); + } + }, this); + }, + + /** + * Loads engines from Chrome URLs asynchronously. + * + * @param aURLs a list of URLs. + * + * @returns {Promise} A promise, resolved successfully if loading data + * succeeds. + */ + _asyncLoadFromChromeURLs: Task.async(function* (aURLs) { + let engines = []; + for (let url of aURLs) { + try { + LOG("_asyncLoadFromChromeURLs: loading engine from chrome url: " + url); + let uri = NetUtil.newURI(url); + let engine = new Engine(uri, true); + yield checkForSyncCompletion(engine._asyncInitFromURI(uri)); + engines.push(engine); + } catch (ex) { + if (ex.result == Cr.NS_ERROR_ALREADY_INITIALIZED) { + throw ex; + } + LOG("_asyncLoadFromChromeURLs: failed to load engine: " + ex); + } + } + return engines; + }), + + _convertChannelToFile: function(chan) { + let fileURI = chan.URI; + while (fileURI instanceof Ci.nsIJARURI) + fileURI = fileURI.JARFile; + fileURI.QueryInterface(Ci.nsIFileURL); + + return fileURI.file; + }, + + _findJAREngines: function SRCH_SVC_findJAREngines() { + LOG("_findJAREngines: looking for engines in JARs") + + let chan = makeChannel(APP_SEARCH_PREFIX + "list.json"); + if (!chan) { + LOG("_findJAREngines: " + APP_SEARCH_PREFIX + " isn't registered"); + return []; + } + + let uris = []; + + let sis = Cc["@mozilla.org/scriptableinputstream;1"]. + createInstance(Ci.nsIScriptableInputStream); + try { + sis.init(chan.open2()); + this._parseListJSON(sis.read(sis.available()), uris); + // parseListJSON will catch its own errors, so we + // should only go into this catch if list.json + // doesn't exist + } catch (e) { + chan = makeChannel(APP_SEARCH_PREFIX + "list.txt"); + sis.init(chan.open2()); + this._parseListTxt(sis.read(sis.available()), uris); + } + return uris; + }, + + /** + * Loads jar engines asynchronously. + * + * @returns {Promise} A promise, resolved successfully if finding jar engines + * succeeds. + */ + _asyncFindJAREngines: Task.async(function* () { + LOG("_asyncFindJAREngines: looking for engines in JARs") + + let listURL = APP_SEARCH_PREFIX + "list.json"; + let chan = makeChannel(listURL); + if (!chan) { + LOG("_asyncFindJAREngines: " + APP_SEARCH_PREFIX + " isn't registered"); + return []; + } + + let uris = []; + + // Read list.json to find the engines we need to load. + let deferred = Promise.defer(); + let request = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"]. + createInstance(Ci.nsIXMLHttpRequest); + request.overrideMimeType("text/plain"); + request.onload = function(aEvent) { + deferred.resolve(aEvent.target.responseText); + }; + request.onerror = function(aEvent) { + LOG("_asyncFindJAREngines: failed to read " + listURL); + // Couldn't find list.json, try list.txt + request.onerror = function(aEvent) { + LOG("_asyncFindJAREngines: failed to read " + APP_SEARCH_PREFIX + "list.txt"); + deferred.resolve(""); + } + request.open("GET", NetUtil.newURI(APP_SEARCH_PREFIX + "list.txt").spec, true); + request.send(); + }; + request.open("GET", NetUtil.newURI(listURL).spec, true); + request.send(); + let list = yield deferred.promise; + + if (request.responseURL.endsWith(".txt")) { + this._parseListTxt(list, uris); + } else { + this._parseListJSON(list, uris); + } + return uris; + }), + + _parseListJSON: function SRCH_SVC_parseListJSON(list, uris) { + let searchSettings; + try { + searchSettings = JSON.parse(list); + } catch (e) { + LOG("failing to parse list.json: " + e); + return; + } + + let jarNames = new Set(); + for (let region in searchSettings) { + // Artifact builds use the full list.json which parses + // slightly differently + if (!("visibleDefaultEngines" in searchSettings[region])) { + continue; + } + for (let engine of searchSettings[region]["visibleDefaultEngines"]) { + jarNames.add(engine); + } + } + + // Check if we have a useable country specific list of visible default engines. + let engineNames; + let visibleDefaultEngines = this.getVerifiedGlobalAttr("visibleDefaultEngines"); + if (visibleDefaultEngines) { + engineNames = visibleDefaultEngines.split(","); + for (let engineName of engineNames) { + // If all engineName values are part of jarNames, + // then we can use the country specific list, otherwise ignore it. + // The visibleDefaultEngines string containing the name of an engine we + // don't ship indicates the server is misconfigured to answer requests + // from the specific Firefox version we are running, so ignoring the + // value altogether is safer. + if (!jarNames.has(engineName)) { + LOG("_parseListJSON: ignoring visibleDefaultEngines value because " + + engineName + " is not in the jar engines we have found"); + engineNames = null; + break; + } + } + } + + // Fallback to building a list based on the regions in the JSON + if (!engineNames || !engineNames.length) { + let region; + if (Services.prefs.prefHasUserValue("browser.search.region")) { + region = Services.prefs.getCharPref("browser.search.region"); + } + if (!region || !(region in searchSettings)) { + region = "default"; + } + engineNames = searchSettings[region]["visibleDefaultEngines"]; + } + + for (let name of engineNames) { + uris.push(APP_SEARCH_PREFIX + name + ".xml"); + } + + // Store this so that it can be used while writing the cache file. + this._visibleDefaultEngines = engineNames; + }, + + _parseListTxt: function SRCH_SVC_parseListTxt(list, uris) { + let names = list.split("\n").filter(n => !!n); + // This maps the names of our built-in engines to a boolean + // indicating whether it should be hidden by default. + let jarNames = new Map(); + for (let name of names) { + if (name.endsWith(":hidden")) { + name = name.split(":")[0]; + jarNames.set(name, true); + } else { + jarNames.set(name, false); + } + } + + // Check if we have a useable country specific list of visible default engines. + let engineNames; + let visibleDefaultEngines = this.getVerifiedGlobalAttr("visibleDefaultEngines"); + if (visibleDefaultEngines) { + engineNames = visibleDefaultEngines.split(","); + + for (let engineName of engineNames) { + // If all engineName values are part of jarNames, + // then we can use the country specific list, otherwise ignore it. + // The visibleDefaultEngines string containing the name of an engine we + // don't ship indicates the server is misconfigured to answer requests + // from the specific Firefox version we are running, so ignoring the + // value altogether is safer. + if (!jarNames.has(engineName)) { + LOG("_parseListTxt: ignoring visibleDefaultEngines value because " + + engineName + " is not in the jar engines we have found"); + engineNames = null; + break; + } + } + } + + // Fallback to building a list based on the :hidden suffixes found in list.txt. + if (!engineNames) { + engineNames = []; + for (let [name, hidden] of jarNames) { + if (!hidden) + engineNames.push(name); + } + } + + for (let name of engineNames) { + uris.push(APP_SEARCH_PREFIX + name + ".xml"); + } + + // Store this so that it can be used while writing the cache file. + this._visibleDefaultEngines = engineNames; + }, + + + _saveSortedEngineList: function SRCH_SVC_saveSortedEngineList() { + LOG("SRCH_SVC_saveSortedEngineList: starting"); + + // Set the useDB pref to indicate that from now on we should use the order + // information stored in the database. + Services.prefs.setBoolPref(BROWSER_SEARCH_PREF + "useDBForOrder", true); + + var engines = this._getSortedEngines(true); + + for (var i = 0; i < engines.length; ++i) { + engines[i].setAttr("order", i + 1); + } + + LOG("SRCH_SVC_saveSortedEngineList: done"); + }, + + _buildSortedEngineList: function SRCH_SVC_buildSortedEngineList() { + LOG("_buildSortedEngineList: building list"); + var addedEngines = { }; + this.__sortedEngines = []; + var engine; + + // If the user has specified a custom engine order, read the order + // information from the metadata instead of the default prefs. + if (getBoolPref(BROWSER_SEARCH_PREF + "useDBForOrder", false)) { + LOG("_buildSortedEngineList: using db for order"); + + // Flag to keep track of whether or not we need to call _saveSortedEngineList. + let needToSaveEngineList = false; + + for (let name in this._engines) { + let engine = this._engines[name]; + var orderNumber = engine.getAttr("order"); + + // Since the DB isn't regularly cleared, and engine files may disappear + // without us knowing, we may already have an engine in this slot. If + // that happens, we just skip it - it will be added later on as an + // unsorted engine. + if (orderNumber && !this.__sortedEngines[orderNumber-1]) { + this.__sortedEngines[orderNumber-1] = engine; + addedEngines[engine.name] = engine; + } else { + // We need to call _saveSortedEngineList so this gets sorted out. + needToSaveEngineList = true; + } + } + + // Filter out any nulls for engines that may have been removed + var filteredEngines = this.__sortedEngines.filter(function(a) { return !!a; }); + if (this.__sortedEngines.length != filteredEngines.length) + needToSaveEngineList = true; + this.__sortedEngines = filteredEngines; + + if (needToSaveEngineList) + this._saveSortedEngineList(); + } else { + // The DB isn't being used, so just read the engine order from the prefs + var i = 0; + var engineName; + var prefName; + + try { + var extras = + Services.prefs.getChildList(BROWSER_SEARCH_PREF + "order.extra."); + + for (prefName of extras) { + engineName = Services.prefs.getCharPref(prefName); + + engine = this._engines[engineName]; + if (!engine || engine.name in addedEngines) + continue; + + this.__sortedEngines.push(engine); + addedEngines[engine.name] = engine; + } + } + catch (e) { } + + let prefNameBase = getGeoSpecificPrefName(BROWSER_SEARCH_PREF + "order"); + while (true) { + prefName = prefNameBase + "." + (++i); + engineName = getLocalizedPref(prefName); + if (!engineName) + break; + + engine = this._engines[engineName]; + if (!engine || engine.name in addedEngines) + continue; + + this.__sortedEngines.push(engine); + addedEngines[engine.name] = engine; + } + } + + // Array for the remaining engines, alphabetically sorted. + let alphaEngines = []; + + for (let name in this._engines) { + let engine = this._engines[name]; + if (!(engine.name in addedEngines)) + alphaEngines.push(this._engines[engine.name]); + } + + let locale = Cc["@mozilla.org/intl/nslocaleservice;1"] + .getService(Ci.nsILocaleService) + .newLocale(getLocale()); + let collation = Cc["@mozilla.org/intl/collation-factory;1"] + .createInstance(Ci.nsICollationFactory) + .CreateCollation(locale); + const strength = Ci.nsICollation.kCollationCaseInsensitiveAscii; + let comparator = (a, b) => collation.compareString(strength, a.name, b.name); + alphaEngines.sort(comparator); + return this.__sortedEngines = this.__sortedEngines.concat(alphaEngines); + }, + + /** + * Get a sorted array of engines. + * @param aWithHidden + * True if hidden plugins should be included in the result. + */ + _getSortedEngines: function SRCH_SVC_getSorted(aWithHidden) { + if (aWithHidden) + return this._sortedEngines; + + return this._sortedEngines.filter(function (engine) { + return !engine.hidden; + }); + }, + + // nsIBrowserSearchService + init: function SRCH_SVC_init(observer) { + LOG("SearchService.init"); + let self = this; + if (!this._initStarted) { + TelemetryStopwatch.start("SEARCH_SERVICE_INIT_MS"); + this._initStarted = true; + Task.spawn(function* task() { + try { + // Complete initialization by calling asynchronous initializer. + yield self._asyncInit(); + TelemetryStopwatch.finish("SEARCH_SERVICE_INIT_MS"); + } catch (ex) { + if (ex.result == Cr.NS_ERROR_ALREADY_INITIALIZED) { + // No need to pursue asynchronous because synchronous fallback was + // called and has finished. + TelemetryStopwatch.finish("SEARCH_SERVICE_INIT_MS"); + } else { + self._initObservers.reject(ex); + TelemetryStopwatch.cancel("SEARCH_SERVICE_INIT_MS"); + } + } + }); + } + if (observer) { + this._initObservers.promise.then( + function onSuccess() { + try { + observer.onInitComplete(self._initRV); + } catch (e) { + Cu.reportError(e); + } + }, + function onError(aReason) { + Cu.reportError("Internal error while initializing SearchService: " + aReason); + observer.onInitComplete(Components.results.NS_ERROR_UNEXPECTED); + } + ); + } + }, + + get isInitialized() { + return gInitialized; + }, + + getEngines: function SRCH_SVC_getEngines(aCount) { + this._ensureInitialized(); + LOG("getEngines: getting all engines"); + var engines = this._getSortedEngines(true); + aCount.value = engines.length; + return engines; + }, + + getVisibleEngines: function SRCH_SVC_getVisible(aCount) { + this._ensureInitialized(); + LOG("getVisibleEngines: getting all visible engines"); + var engines = this._getSortedEngines(false); + aCount.value = engines.length; + return engines; + }, + + getDefaultEngines: function SRCH_SVC_getDefault(aCount) { + this._ensureInitialized(); + function isDefault(engine) { + return engine._isDefault; + } + var engines = this._sortedEngines.filter(isDefault); + var engineOrder = {}; + var engineName; + var i = 1; + + // Build a list of engines which we have ordering information for. + // We're rebuilding the list here because _sortedEngines contain the + // current order, but we want the original order. + + // First, look at the "browser.search.order.extra" branch. + try { + var extras = Services.prefs.getChildList(BROWSER_SEARCH_PREF + "order.extra."); + + for (var prefName of extras) { + engineName = Services.prefs.getCharPref(prefName); + + if (!(engineName in engineOrder)) + engineOrder[engineName] = i++; + } + } catch (e) { + LOG("Getting extra order prefs failed: " + e); + } + + // Now look through the "browser.search.order" branch. + let prefNameBase = getGeoSpecificPrefName(BROWSER_SEARCH_PREF + "order"); + for (var j = 1; ; j++) { + let prefName = prefNameBase + "." + j; + engineName = getLocalizedPref(prefName); + if (!engineName) + break; + + if (!(engineName in engineOrder)) + engineOrder[engineName] = i++; + } + + LOG("getDefaultEngines: engineOrder: " + engineOrder.toSource()); + + function compareEngines (a, b) { + var aIdx = engineOrder[a.name]; + var bIdx = engineOrder[b.name]; + + if (aIdx && bIdx) + return aIdx - bIdx; + if (aIdx) + return -1; + if (bIdx) + return 1; + + return a.name.localeCompare(b.name); + } + engines.sort(compareEngines); + + aCount.value = engines.length; + return engines; + }, + + getEngineByName: function SRCH_SVC_getEngineByName(aEngineName) { + this._ensureInitialized(); + return this._engines[aEngineName] || null; + }, + + getEngineByAlias: function SRCH_SVC_getEngineByAlias(aAlias) { + this._ensureInitialized(); + for (var engineName in this._engines) { + var engine = this._engines[engineName]; + if (engine && engine.alias == aAlias) + return engine; + } + return null; + }, + + addEngineWithDetails: function SRCH_SVC_addEWD(aName, aIconURL, aAlias, + aDescription, aMethod, + aTemplate, aExtensionID) { + this._ensureInitialized(); + if (!aName) + FAIL("Invalid name passed to addEngineWithDetails!"); + if (!aMethod) + FAIL("Invalid method passed to addEngineWithDetails!"); + if (!aTemplate) + FAIL("Invalid template passed to addEngineWithDetails!"); + if (this._engines[aName]) + FAIL("An engine with that name already exists!", Cr.NS_ERROR_FILE_ALREADY_EXISTS); + + var engine = new Engine(sanitizeName(aName), false); + engine._initFromMetadata(aName, aIconURL, aAlias, aDescription, + aMethod, aTemplate, aExtensionID); + engine._loadPath = "[other]addEngineWithDetails"; + this._addEngineToStore(engine); + }, + + addEngine: function SRCH_SVC_addEngine(aEngineURL, aDataType, aIconURL, + aConfirm, aCallback) { + LOG("addEngine: Adding \"" + aEngineURL + "\"."); + this._ensureInitialized(); + try { + var uri = makeURI(aEngineURL); + var engine = new Engine(uri, false); + if (aCallback) { + engine._installCallback = function (errorCode) { + try { + if (errorCode == null) + aCallback.onSuccess(engine); + else + aCallback.onError(errorCode); + } catch (ex) { + Cu.reportError("Error invoking addEngine install callback: " + ex); + } + // Clear the reference to the callback now that it's been invoked. + engine._installCallback = null; + }; + } + engine._initFromURIAndLoad(uri); + } catch (ex) { + // Drop the reference to the callback, if set + if (engine) + engine._installCallback = null; + FAIL("addEngine: Error adding engine:\n" + ex, Cr.NS_ERROR_FAILURE); + } + engine._setIcon(aIconURL, false); + engine._confirm = aConfirm; + }, + + removeEngine: function SRCH_SVC_removeEngine(aEngine) { + this._ensureInitialized(); + if (!aEngine) + FAIL("no engine passed to removeEngine!"); + + var engineToRemove = null; + for (var e in this._engines) { + if (aEngine.wrappedJSObject == this._engines[e]) + engineToRemove = this._engines[e]; + } + + if (!engineToRemove) + FAIL("removeEngine: Can't find engine to remove!", Cr.NS_ERROR_FILE_NOT_FOUND); + + if (engineToRemove == this.currentEngine) { + this._currentEngine = null; + } + + if (engineToRemove._readOnly) { + // Just hide it (the "hidden" setter will notify) and remove its alias to + // avoid future conflicts with other engines. + engineToRemove.hidden = true; + engineToRemove.alias = null; + } else { + // Remove the engine file from disk if we had a legacy file in the profile. + if (engineToRemove._filePath) { + let file = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsILocalFile); + file.persistentDescriptor = engineToRemove._filePath; + if (file.exists()) { + file.remove(false); + } + engineToRemove._filePath = null; + } + + // Remove the engine from _sortedEngines + var index = this._sortedEngines.indexOf(engineToRemove); + if (index == -1) + FAIL("Can't find engine to remove in _sortedEngines!", Cr.NS_ERROR_FAILURE); + this.__sortedEngines.splice(index, 1); + + // Remove the engine from the internal store + delete this._engines[engineToRemove.name]; + + // Since we removed an engine, we need to update the preferences. + this._saveSortedEngineList(); + } + notifyAction(engineToRemove, SEARCH_ENGINE_REMOVED); + }, + + moveEngine: function SRCH_SVC_moveEngine(aEngine, aNewIndex) { + this._ensureInitialized(); + if ((aNewIndex > this._sortedEngines.length) || (aNewIndex < 0)) + FAIL("SRCH_SVC_moveEngine: Index out of bounds!"); + if (!(aEngine instanceof Ci.nsISearchEngine)) + FAIL("SRCH_SVC_moveEngine: Invalid engine passed to moveEngine!"); + if (aEngine.hidden) + FAIL("moveEngine: Can't move a hidden engine!", Cr.NS_ERROR_FAILURE); + + var engine = aEngine.wrappedJSObject; + + var currentIndex = this._sortedEngines.indexOf(engine); + if (currentIndex == -1) + FAIL("moveEngine: Can't find engine to move!", Cr.NS_ERROR_UNEXPECTED); + + // Our callers only take into account non-hidden engines when calculating + // aNewIndex, but we need to move it in the array of all engines, so we + // need to adjust aNewIndex accordingly. To do this, we count the number + // of hidden engines in the list before the engine that we're taking the + // place of. We do this by first finding newIndexEngine (the engine that + // we were supposed to replace) and then iterating through the complete + // engine list until we reach it, increasing aNewIndex for each hidden + // engine we find on our way there. + // + // This could be further simplified by having our caller pass in + // newIndexEngine directly instead of aNewIndex. + var newIndexEngine = this._getSortedEngines(false)[aNewIndex]; + if (!newIndexEngine) + FAIL("moveEngine: Can't find engine to replace!", Cr.NS_ERROR_UNEXPECTED); + + for (var i = 0; i < this._sortedEngines.length; ++i) { + if (newIndexEngine == this._sortedEngines[i]) + break; + if (this._sortedEngines[i].hidden) + aNewIndex++; + } + + if (currentIndex == aNewIndex) + return; // nothing to do! + + // Move the engine + var movedEngine = this.__sortedEngines.splice(currentIndex, 1)[0]; + this.__sortedEngines.splice(aNewIndex, 0, movedEngine); + + notifyAction(engine, SEARCH_ENGINE_CHANGED); + + // Since we moved an engine, we need to update the preferences. + this._saveSortedEngineList(); + }, + + restoreDefaultEngines: function SRCH_SVC_resetDefaultEngines() { + this._ensureInitialized(); + for (let name in this._engines) { + let e = this._engines[name]; + // Unhide all default engines + if (e.hidden && e._isDefault) + e.hidden = false; + } + }, + + get defaultEngine() { return this.currentEngine; }, + + set defaultEngine(val) { + this.currentEngine = val; + }, + + get currentEngine() { + this._ensureInitialized(); + if (!this._currentEngine) { + let name = this.getGlobalAttr("current"); + let engine = this.getEngineByName(name); + if (engine && (this.getGlobalAttr("hash") == getVerificationHash(name) || + engine._isDefault)) { + // If the current engine is a default one, we can relax the + // verification hash check to reduce the annoyance for users who + // backup/sync their profile in custom ways. + this._currentEngine = engine; + } + if (!name) + this._currentEngine = this.originalDefaultEngine; + } + + // If the current engine is not set or hidden, we fallback... + if (!this._currentEngine || this._currentEngine.hidden) { + // first to the original default engine + let originalDefault = this.originalDefaultEngine; + if (!originalDefault || originalDefault.hidden) { + // then to the first visible engine + let firstVisible = this._getSortedEngines(false)[0]; + if (firstVisible && !firstVisible.hidden) { + this.currentEngine = firstVisible; + return firstVisible; + } + // and finally as a last resort we unhide the original default engine. + if (originalDefault) + originalDefault.hidden = false; + } + if (!originalDefault) + return null; + + // If the current engine wasn't set or was hidden, we used a fallback + // to pick a new current engine. As soon as we return it, this new + // current engine will become user-visible, so we should persist it. + // by calling the setter. + this.currentEngine = originalDefault; + } + + return this._currentEngine; + }, + + set currentEngine(val) { + this._ensureInitialized(); + // Sometimes we get wrapped nsISearchEngine objects (external XPCOM callers), + // and sometimes we get raw Engine JS objects (callers in this file), so + // handle both. + if (!(val instanceof Ci.nsISearchEngine) && !(val instanceof Engine)) + FAIL("Invalid argument passed to currentEngine setter"); + + var newCurrentEngine = this.getEngineByName(val.name); + if (!newCurrentEngine) + FAIL("Can't find engine in store!", Cr.NS_ERROR_UNEXPECTED); + + if (!newCurrentEngine._isDefault) { + // If a non default engine is being set as the current engine, ensure + // its loadPath has a verification hash. + if (!newCurrentEngine._loadPath) + newCurrentEngine._loadPath = "[other]unknown"; + let loadPathHash = getVerificationHash(newCurrentEngine._loadPath); + let currentHash = newCurrentEngine.getAttr("loadPathHash"); + if (!currentHash || currentHash != loadPathHash) { + newCurrentEngine.setAttr("loadPathHash", loadPathHash); + notifyAction(newCurrentEngine, SEARCH_ENGINE_CHANGED); + } + } + + if (newCurrentEngine == this._currentEngine) + return; + + this._currentEngine = newCurrentEngine; + + // If we change the default engine in the future, that change should impact + // users who have switched away from and then back to the build's "default" + // engine. So clear the user pref when the currentEngine is set to the + // build's default engine, so that the currentEngine getter falls back to + // whatever the default is. + let newName = this._currentEngine.name; + if (this._currentEngine == this.originalDefaultEngine) { + newName = ""; + } + + this.setGlobalAttr("current", newName); + this.setGlobalAttr("hash", getVerificationHash(newName)); + + notifyAction(this._currentEngine, SEARCH_ENGINE_DEFAULT); + notifyAction(this._currentEngine, SEARCH_ENGINE_CURRENT); + }, + + getDefaultEngineInfo() { + let result = {}; + + let engine; + try { + engine = this.defaultEngine; + } catch (e) { + // The defaultEngine getter will throw if there's no engine at all, + // which shouldn't happen unless an add-on or a test deleted all of them. + // Our preferences UI doesn't let users do that. + Cu.reportError("getDefaultEngineInfo: No default engine"); + } + + if (!engine) { + result.name = "NONE"; + } else { + if (engine.name) + result.name = engine.name; + + result.loadPath = engine._loadPath; + + let origin; + if (engine._isDefault) + origin = "default"; + else { + let currentHash = engine.getAttr("loadPathHash"); + if (!currentHash) + origin = "unverified"; + else { + let loadPathHash = getVerificationHash(engine._loadPath); + origin = currentHash == loadPathHash ? "verified" : "invalid"; + } + } + result.origin = origin; + + // For privacy, we only collect the submission URL for default engines... + let sendSubmissionURL = engine._isDefault; + + // ... or engines sorted by default near the top of the list. + if (!sendSubmissionURL) { + let extras = + Services.prefs.getChildList(BROWSER_SEARCH_PREF + "order.extra."); + + for (let prefName of extras) { + try { + if (result.name == Services.prefs.getCharPref(prefName)) { + sendSubmissionURL = true; + break; + } + } catch (e) {} + } + + let prefNameBase = getGeoSpecificPrefName(BROWSER_SEARCH_PREF + "order"); + let i = 0; + while (!sendSubmissionURL) { + let prefName = prefNameBase + "." + (++i); + let engineName = getLocalizedPref(prefName); + if (!engineName) + break; + if (result.name == engineName) { + sendSubmissionURL = true; + break; + } + } + } + + if (sendSubmissionURL) { + let uri = engine._getURLOfType("text/html") + .getSubmission("", engine, "searchbar").uri; + uri.userPass = ""; // Avoid reporting a username or password. + result.submissionURL = uri.spec; + } + } + + return result; + }, + + _recordEngineTelemetry: function() { + Services.telemetry.getHistogramById("SEARCH_SERVICE_ENGINE_COUNT") + .add(Object.keys(this._engines).length); + let hasUpdates = false; + let hasIconUpdates = false; + for (let name in this._engines) { + let engine = this._engines[name]; + if (engine._hasUpdates) { + hasUpdates = true; + if (engine._iconUpdateURL) { + hasIconUpdates = true; + break; + } + } + } + Services.telemetry.getHistogramById("SEARCH_SERVICE_HAS_UPDATES").add(hasUpdates); + Services.telemetry.getHistogramById("SEARCH_SERVICE_HAS_ICON_UPDATES").add(hasIconUpdates); + }, + + /** + * This map is built lazily after the available search engines change. It + * allows quick parsing of an URL representing a search submission into the + * search engine name and original terms. + * + * The keys are strings containing the domain name and lowercase path of the + * engine submission, for example "www.google.com/search". + * + * The values are objects with these properties: + * { + * engine: The associated nsISearchEngine. + * termsParameterName: Name of the URL parameter containing the search + * terms, for example "q". + * } + */ + _parseSubmissionMap: null, + + _buildParseSubmissionMap: function SRCH_SVC__buildParseSubmissionMap() { + LOG("_buildParseSubmissionMap"); + this._parseSubmissionMap = new Map(); + + // Used only while building the map, indicates which entries do not refer to + // the main domain of the engine but to an alternate domain, for example + // "www.google.fr" for the "www.google.com" search engine. + let keysOfAlternates = new Set(); + + for (let engine of this._sortedEngines) { + LOG("Processing engine: " + engine.name); + + if (engine.hidden) { + LOG("Engine is hidden."); + continue; + } + + let urlParsingInfo = engine.getURLParsingInfo(); + if (!urlParsingInfo) { + LOG("Engine does not support URL parsing."); + continue; + } + + // Store the same object on each matching map key, as an optimization. + let mapValueForEngine = { + engine: engine, + termsParameterName: urlParsingInfo.termsParameterName, + }; + + let processDomain = (domain, isAlternate) => { + let key = domain + urlParsingInfo.path; + + // Apply the logic for which main domains take priority over alternate + // domains, even if they are found later in the ordered engine list. + let existingEntry = this._parseSubmissionMap.get(key); + if (!existingEntry) { + LOG("Adding new entry: " + key); + if (isAlternate) { + keysOfAlternates.add(key); + } + } else if (!isAlternate && keysOfAlternates.has(key)) { + LOG("Overriding alternate entry: " + key + + " (" + existingEntry.engine.name + ")"); + keysOfAlternates.delete(key); + } else { + LOG("Keeping existing entry: " + key + + " (" + existingEntry.engine.name + ")"); + return; + } + + this._parseSubmissionMap.set(key, mapValueForEngine); + }; + + processDomain(urlParsingInfo.mainDomain, false); + SearchStaticData.getAlternateDomains(urlParsingInfo.mainDomain) + .forEach(d => processDomain(d, true)); + } + }, + + /** + * Checks to see if any engine has an EngineURL of type URLTYPE_SEARCH_HTML + * for this request-method, template URL, and query params. + */ + hasEngineWithURL: function(method, template, formData) { + this._ensureInitialized(); + + // Quick helper method to ensure formData filtered/sorted for compares. + let getSortedFormData = data => { + return data.filter(a => a.name && a.value).sort((a, b) => { + if (a.name > b.name) { + return 1; + } else if (b.name > a.name) { + return -1; + } else if (a.value > b.value) { + return 1; + } + return (b.value > a.value) ? -1 : 0; + }); + }; + + // Sanitize method, ensure formData is pre-sorted. + let methodUpper = method.toUpperCase(); + let sortedFormData = getSortedFormData(formData); + let sortedFormLength = sortedFormData.length; + + return this._getSortedEngines(false).some(engine => { + return engine._urls.some(url => { + // Not an engineURL match if type, method, url, #params don't match. + if (url.type != URLTYPE_SEARCH_HTML || + url.method != methodUpper || + url.template != template || + url.params.length != sortedFormLength) { + return false; + } + + // Ensure engineURL formData is pre-sorted. Then, we're + // not an engineURL match if any queryParam doesn't compare. + let sortedParams = getSortedFormData(url.params); + for (let i = 0; i < sortedFormLength; i++) { + let formData = sortedFormData[i]; + let param = sortedParams[i]; + if (param.name != formData.name || + param.value != formData.value || + param.purpose != formData.purpose) { + return false; + } + } + // Else we're a match. + return true; + }); + }); + }, + + parseSubmissionURL: function SRCH_SVC_parseSubmissionURL(aURL) { + this._ensureInitialized(); + LOG("parseSubmissionURL: Parsing \"" + aURL + "\"."); + + if (!this._parseSubmissionMap) { + this._buildParseSubmissionMap(); + } + + // Extract the elements of the provided URL first. + let soughtKey, soughtQuery; + try { + let soughtUrl = NetUtil.newURI(aURL).QueryInterface(Ci.nsIURL); + + // Exclude any URL that is not HTTP or HTTPS from the beginning. + if (soughtUrl.scheme != "http" && soughtUrl.scheme != "https") { + LOG("The URL scheme is not HTTP or HTTPS."); + return gEmptyParseSubmissionResult; + } + + // Reading these URL properties may fail and raise an exception. + soughtKey = soughtUrl.host + soughtUrl.filePath.toLowerCase(); + soughtQuery = soughtUrl.query; + } catch (ex) { + // Errors while parsing the URL or accessing the properties are not fatal. + LOG("The value does not look like a structured URL."); + return gEmptyParseSubmissionResult; + } + + // Look up the domain and path in the map to identify the search engine. + let mapEntry = this._parseSubmissionMap.get(soughtKey); + if (!mapEntry) { + LOG("No engine associated with domain and path: " + soughtKey); + return gEmptyParseSubmissionResult; + } + + // Extract the search terms from the parameter, for example "caff%C3%A8" + // from the URL "https://www.google.com/search?q=caff%C3%A8&client=firefox". + let encodedTerms = null; + for (let param of soughtQuery.split("&")) { + let equalPos = param.indexOf("="); + if (equalPos != -1 && + param.substr(0, equalPos) == mapEntry.termsParameterName) { + // This is the parameter we are looking for. + encodedTerms = param.substr(equalPos + 1); + break; + } + } + if (encodedTerms === null) { + LOG("Missing terms parameter: " + mapEntry.termsParameterName); + return gEmptyParseSubmissionResult; + } + + let length = 0; + let offset = aURL.indexOf("?") + 1; + let query = aURL.slice(offset); + // Iterate a second time over the original input string to determine the + // correct search term offset and length in the original encoding. + for (let param of query.split("&")) { + let equalPos = param.indexOf("="); + if (equalPos != -1 && + param.substr(0, equalPos) == mapEntry.termsParameterName) { + // This is the parameter we are looking for. + offset += equalPos + 1; + length = param.length - equalPos - 1; + break; + } + offset += param.length + 1; + } + + // Decode the terms using the charset defined in the search engine. + let terms; + try { + terms = gTextToSubURI.UnEscapeAndConvert( + mapEntry.engine.queryCharset, + encodedTerms.replace(/\+/g, " ")); + } catch (ex) { + // Decoding errors will cause this match to be ignored. + LOG("Parameter decoding failed. Charset: " + + mapEntry.engine.queryCharset); + return gEmptyParseSubmissionResult; + } + + LOG("Match found. Terms: " + terms); + return new ParseSubmissionResult(mapEntry.engine, terms, offset, length); + }, + + // nsIObserver + observe: function SRCH_SVC_observe(aEngine, aTopic, aVerb) { + switch (aTopic) { + case SEARCH_ENGINE_TOPIC: + switch (aVerb) { + case SEARCH_ENGINE_LOADED: + var engine = aEngine.QueryInterface(Ci.nsISearchEngine); + LOG("nsSearchService::observe: Done installation of " + engine.name + + "."); + this._addEngineToStore(engine.wrappedJSObject); + if (engine.wrappedJSObject._useNow) { + LOG("nsSearchService::observe: setting current"); + this.currentEngine = aEngine; + } + // The addition of the engine to the store always triggers an ADDED + // or a CHANGED notification, that will trigger the task below. + break; + case SEARCH_ENGINE_ADDED: + case SEARCH_ENGINE_CHANGED: + case SEARCH_ENGINE_REMOVED: + this.batchTask.disarm(); + this.batchTask.arm(); + // Invalidate the map used to parse URLs to search engines. + this._parseSubmissionMap = null; + break; + } + break; + + case QUIT_APPLICATION_TOPIC: + this._removeObservers(); + break; + + case "nsPref:changed": + if (aVerb == LOCALE_PREF) { + // Locale changed. Re-init. We rely on observers, because we can't + // return this promise to anyone. + this._asyncReInit(); + break; + } + } + }, + + // nsITimerCallback + notify: function SRCH_SVC_notify(aTimer) { + LOG("_notify: checking for updates"); + + if (!getBoolPref(BROWSER_SEARCH_PREF + "update", true)) + return; + + // Our timer has expired, but unfortunately, we can't get any data from it. + // Therefore, we need to walk our engine-list, looking for expired engines + var currentTime = Date.now(); + LOG("currentTime: " + currentTime); + for (let name in this._engines) { + let engine = this._engines[name].wrappedJSObject; + if (!engine._hasUpdates) + continue; + + LOG("checking " + engine.name); + + var expirTime = engine.getAttr("updateexpir"); + LOG("expirTime: " + expirTime + "\nupdateURL: " + engine._updateURL + + "\niconUpdateURL: " + engine._iconUpdateURL); + + var engineExpired = expirTime <= currentTime; + + if (!expirTime || !engineExpired) { + LOG("skipping engine"); + continue; + } + + LOG(engine.name + " has expired"); + + engineUpdateService.update(engine); + + // Schedule the next update + engineUpdateService.scheduleNextUpdate(engine); + + } // end engine iteration + }, + + _addObservers: function SRCH_SVC_addObservers() { + if (this._observersAdded) { + // There might be a race between synchronous and asynchronous + // initialization for which we try to register the observers twice. + return; + } + this._observersAdded = true; + + Services.obs.addObserver(this, SEARCH_ENGINE_TOPIC, false); + Services.obs.addObserver(this, QUIT_APPLICATION_TOPIC, false); + +#ifdef MOZ_FENNEC + Services.prefs.addObserver(LOCALE_PREF, this, false); +#endif + + // The current stage of shutdown. Used to help analyze crash + // signatures in case of shutdown timeout. + let shutdownState = { + step: "Not started", + latestError: { + message: undefined, + stack: undefined + } + }; + OS.File.profileBeforeChange.addBlocker( + "Search service: shutting down", + () => Task.spawn(function* () { + if (this._batchTask) { + shutdownState.step = "Finalizing batched task"; + try { + yield this._batchTask.finalize(); + shutdownState.step = "Batched task finalized"; + } catch (ex) { + shutdownState.step = "Batched task failed to finalize"; + + shutdownState.latestError.message = "" + ex; + if (ex && typeof ex == "object") { + shutdownState.latestError.stack = ex.stack || undefined; + } + + // Ensure that error is reported and that it causes tests + // to fail. + Promise.reject(ex); + } + } + }.bind(this)), + + () => shutdownState + ); + }, + _observersAdded: false, + + _removeObservers: function SRCH_SVC_removeObservers() { + Services.obs.removeObserver(this, SEARCH_ENGINE_TOPIC); + Services.obs.removeObserver(this, QUIT_APPLICATION_TOPIC); + +#ifdef MOZ_FENNEC + Services.prefs.removeObserver(LOCALE_PREF, this); +#endif + }, + + QueryInterface: XPCOMUtils.generateQI([ + Ci.nsIBrowserSearchService, + Ci.nsIObserver, + Ci.nsITimerCallback + ]) +}; + + +const SEARCH_UPDATE_LOG_PREFIX = "*** Search update: "; + +/** + * Outputs aText to the JavaScript console as well as to stdout, if the search + * logging pref (browser.search.update.log) is set to true. + */ +function ULOG(aText) { + if (getBoolPref(BROWSER_SEARCH_PREF + "update.log", false)) { + dump(SEARCH_UPDATE_LOG_PREFIX + aText + "\n"); + Services.console.logStringMessage(aText); + } +} + +var engineUpdateService = { + scheduleNextUpdate: function eus_scheduleNextUpdate(aEngine) { + var interval = aEngine._updateInterval || SEARCH_DEFAULT_UPDATE_INTERVAL; + var milliseconds = interval * 86400000; // |interval| is in days + aEngine.setAttr("updateexpir", Date.now() + milliseconds); + }, + + update: function eus_Update(aEngine) { + let engine = aEngine.wrappedJSObject; + ULOG("update called for " + aEngine._name); + if (!getBoolPref(BROWSER_SEARCH_PREF + "update", true) || !engine._hasUpdates) + return; + + let testEngine = null; + let updateURL = engine._getURLOfType(URLTYPE_OPENSEARCH); + let updateURI = (updateURL && updateURL._hasRelation("self")) ? + updateURL.getSubmission("", engine).uri : + makeURI(engine._updateURL); + if (updateURI) { + if (engine._isDefault && !updateURI.schemeIs("https")) { + ULOG("Invalid scheme for default engine update"); + return; + } + + ULOG("updating " + engine.name + " from " + updateURI.spec); + testEngine = new Engine(updateURI, false); + testEngine._engineToUpdate = engine; + testEngine._initFromURIAndLoad(updateURI); + } else + ULOG("invalid updateURI"); + + if (engine._iconUpdateURL) { + // If we're updating the engine too, use the new engine object, + // otherwise use the existing engine object. + (testEngine || engine)._setIcon(engine._iconUpdateURL, true); + } + } +}; + +this.NSGetFactory = XPCOMUtils.generateNSGetFactory([SearchService]); |