From 1037746d009ab5c0b20765a8032738ad189dc407 Mon Sep 17 00:00:00 2001 From: wolfbeast Date: Fri, 27 Mar 2020 12:49:01 +0100 Subject: Issue #1498 - Part 1: Stop using HSTS preload lists. --- security/manager/tools/getHSTSPreloadList.js | 461 --------------------------- 1 file changed, 461 deletions(-) delete mode 100644 security/manager/tools/getHSTSPreloadList.js (limited to 'security/manager/tools') diff --git a/security/manager/tools/getHSTSPreloadList.js b/security/manager/tools/getHSTSPreloadList.js deleted file mode 100644 index cf5614651..000000000 --- a/security/manager/tools/getHSTSPreloadList.js +++ /dev/null @@ -1,461 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -"use strict"; - -// How to run this file: -// 1. [obtain firefox source code] -// 2. [build/obtain firefox binaries] -// 3. run `[path to]/run-mozilla.sh [path to]/xpcshell \ -// [path to]/getHSTSPreloadlist.js \ -// [absolute path to]/nsSTSPreloadlist.inc' -// Note: Running this file outputs a new nsSTSPreloadlist.inc in the current -// working directory. - -var Cc = Components.classes; -var Ci = Components.interfaces; -var Cu = Components.utils; -var Cr = Components.results; - -Cu.import("resource://gre/modules/Services.jsm"); -Cu.import("resource://gre/modules/FileUtils.jsm"); -Cu.import("resource:///modules/XPCOMUtils.jsm"); - -const SOURCE = "https://chromium.googlesource.com/chromium/src/net/+/master/http/transport_security_state_static.json?format=TEXT"; -const TOOL_IDENTIFIER = "UXP HSTS preload list verifier" -const OUTPUT = "nsSTSPreloadList.inc"; -const ERROR_OUTPUT = "nsSTSPreloadList.errors"; -const MINIMUM_REQUIRED_MAX_AGE = 60 * 60 * 24 * 7 * 18; -const MAX_CONCURRENT_REQUESTS = 15; -const MAX_RETRIES = 2; -const REQUEST_TIMEOUT = 10 * 1000; -const ERROR_NONE = "no error"; -const ERROR_CONNECTING_TO_HOST = "could not connect to host"; -const ERROR_NO_HSTS_HEADER = "did not receive HSTS header"; -const ERROR_MAX_AGE_TOO_LOW = "max-age too low: "; -const HEADER = "/* This Source Code Form is subject to the terms of the Mozilla Public\n" + -" * License, v. 2.0. If a copy of the MPL was not distributed with this\n" + -" * file, You can obtain one at http://mozilla.org/MPL/2.0/. */\n" + -"\n" + -"/*****************************************************************************/\n" + -"/* This is an automatically generated file. If you're not */\n" + -"/* nsSiteSecurityService.cpp, you shouldn't be #including it. */\n" + -"/*****************************************************************************/\n" + -"\n" + -"#include \n"; -const PREFIX = "\n" + -"class nsSTSPreload\n" + -"{\n" + -" public:\n" + -" const char *mHost;\n" + -" const bool mIncludeSubdomains;\n" + -"};\n" + -"\n" + -"static const nsSTSPreload kSTSPreloadList[] = {\n"; -const POSTFIX = "};\n"; - -function download() { - var req = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"] - .createInstance(Ci.nsIXMLHttpRequest); - req.open("GET", SOURCE, false); // doing the request synchronously - try { - req.send(); - } - catch (e) { - throw new Error(`ERROR: problem downloading '${SOURCE}': ${e}`); - } - - if (req.status != 200) { - throw new Error("ERROR: problem downloading '" + SOURCE + "': status " + - req.status); - } - - var resultDecoded; - try { - resultDecoded = atob(req.responseText); - } - catch (e) { - throw new Error("ERROR: could not decode data as base64 from '" + SOURCE + - "': " + e); - } - - // we have to filter out '//' comments, while not mangling the json - var result = resultDecoded.replace(/^(\s*)?\/\/[^\n]*\n/mg, ""); - var data = null; - try { - data = JSON.parse(result); - } - catch (e) { - throw new Error(`ERROR: could not parse data from '${SOURCE}': ${e}`); - } - return data; -} - -function getHosts(rawdata) { - var hosts = []; - - if (!rawdata || !rawdata.entries) { - throw new Error("ERROR: source data not formatted correctly: 'entries' " + - "not found"); - } - - for (let entry of rawdata.entries) { - if (entry.mode && entry.mode == "force-https") { - if (entry.name) { - // We trim the entry name here to avoid malformed URI exceptions when we - // later try to connect to the domain. - entry.name = entry.name.trim(); - entry.retries = MAX_RETRIES; - entry.originalIncludeSubdomains = entry.include_subdomains; - hosts.push(entry); - } else { - throw new Error("ERROR: entry not formatted correctly: no name found"); - } - } - } - - return hosts; -} - -var gSSService = Cc["@mozilla.org/ssservice;1"] - .getService(Ci.nsISiteSecurityService); - -function processStsHeader(host, header, status, securityInfo) { - var maxAge = { value: 0 }; - var includeSubdomains = { value: false }; - var error = ERROR_NONE; - if (header != null && securityInfo != null) { - try { - var uri = Services.io.newURI("https://" + host.name, null, null); - var sslStatus = securityInfo.QueryInterface(Ci.nsISSLStatusProvider) - .SSLStatus; - gSSService.processHeader(Ci.nsISiteSecurityService.HEADER_HSTS, - uri, header, sslStatus, 0, maxAge, - includeSubdomains); - } - catch (e) { - dump("ERROR: could not process header '" + header + "' from " + - host.name + ": " + e + "\n"); - error = e; - } - } else if (status == 0) { - error = ERROR_CONNECTING_TO_HOST; - } else { - error = ERROR_NO_HSTS_HEADER; - } - - let forceInclude = (host.forceInclude || host.pins == "google"); - - if (error == ERROR_NONE && maxAge.value < MINIMUM_REQUIRED_MAX_AGE) { - error = ERROR_MAX_AGE_TOO_LOW; - } - - return { name: host.name, - maxAge: maxAge.value, - includeSubdomains: includeSubdomains.value, - error: error, - retries: host.retries - 1, - forceInclude: forceInclude, - originalIncludeSubdomains: host.originalIncludeSubdomains }; -} - -// RedirectAndAuthStopper prevents redirects and HTTP authentication -function RedirectAndAuthStopper() {} - -RedirectAndAuthStopper.prototype = { - // nsIChannelEventSink - asyncOnChannelRedirect: function(oldChannel, newChannel, flags, callback) { - throw new Error(Cr.NS_ERROR_ENTITY_CHANGED); - }, - - // nsIAuthPrompt2 - promptAuth: function(channel, level, authInfo) { - return false; - }, - - asyncPromptAuth: function(channel, callback, context, level, authInfo) { - throw new Error(Cr.NS_ERROR_NOT_IMPLEMENTED); - }, - - getInterface: function(iid) { - return this.QueryInterface(iid); - }, - - QueryInterface: XPCOMUtils.generateQI([Ci.nsIChannelEventSink, - Ci.nsIAuthPrompt2]) -}; - -function getHSTSStatus(host, resultList) { - var req = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"] - .createInstance(Ci.nsIXMLHttpRequest); - var inResultList = false; - var uri = "https://" + host.name + "/"; - req.open("HEAD", uri, true); - req.setRequestHeader("X-Automated-Tool", TOOL_IDENTIFIER); - req.timeout = REQUEST_TIMEOUT; - - let errorhandler = (evt) => { - dump(`ERROR: error making request to ${host.name} (type=${evt.type})\n`); - if (!inResultList) { - inResultList = true; - resultList.push(processStsHeader(host, null, req.status, - req.channel.securityInfo)); - } - }; - req.onerror = errorhandler; - req.ontimeout = errorhandler; - req.onabort = errorhandler; - - req.onload = function(event) { - if (!inResultList) { - inResultList = true; - var header = req.getResponseHeader("strict-transport-security"); - resultList.push(processStsHeader(host, header, req.status, - req.channel.securityInfo)); - } - }; - - try { - req.channel.notificationCallbacks = new RedirectAndAuthStopper(); - req.send(); - } - catch (e) { - dump("ERROR: exception making request to " + host.name + ": " + e + "\n"); - } -} - -function compareHSTSStatus(a, b) { - if (a.name > b.name) { - return 1; - } - if (a.name < b.name) { - return -1; - } - return 0; -} - -function writeTo(string, fos) { - fos.write(string, string.length); -} - -// Determines and returns a string representing a declaration of when this -// preload list should no longer be used. -// This is the current time plus MINIMUM_REQUIRED_MAX_AGE. -function getExpirationTimeString() { - var now = new Date(); - var nowMillis = now.getTime(); - // MINIMUM_REQUIRED_MAX_AGE is in seconds, so convert to milliseconds - var expirationMillis = nowMillis + (MINIMUM_REQUIRED_MAX_AGE * 1000); - var expirationMicros = expirationMillis * 1000; - return "const PRTime gPreloadListExpirationTime = INT64_C(" + expirationMicros + ");\n"; -} - -function errorToString(status) { - return (status.error == ERROR_MAX_AGE_TOO_LOW - ? status.error + status.maxAge - : status.error); -} - -function writeEntry(status, outputStream) { - let incSubdomainsBool = (status.forceInclude && status.error != ERROR_NONE - ? status.originalIncludeSubdomains - : status.includeSubdomains); - let includeSubdomains = (incSubdomainsBool ? "true" : "false"); - writeTo(" { \"" + status.name + "\", " + includeSubdomains + " },\n", - outputStream); -} - -function output(sortedStatuses, currentList) { - try { - var file = FileUtils.getFile("CurWorkD", [OUTPUT]); - var errorFile = FileUtils.getFile("CurWorkD", [ERROR_OUTPUT]); - var fos = FileUtils.openSafeFileOutputStream(file); - var eos = FileUtils.openSafeFileOutputStream(errorFile); - writeTo(HEADER, fos); - writeTo(getExpirationTimeString(), fos); - writeTo(PREFIX, fos); - - dump("INFO: Removing error-state sites from list\n"); - for (let status in sortedStatuses) { - // If we've encountered an error for this entry (other than the site not - // sending an HSTS header), be safe and remove it from the list - // (preventing stale entries from accumulating). - if (status.error != ERROR_NONE && - status.error != ERROR_NO_HSTS_HEADER && - status.error != ERROR_MAX_AGE_TOO_LOW && - status.name in currentList) { - // dump("INFO: error connecting to or processing " + status.name + " - dropping from list\n"); - writeTo(status.name + ": " + errorToString(status) + "\n", eos); - status.maxAge = 0; - } - } - - // Filter out entries we aren't including. - dump("INFO: Filtering out entries we aren't including...\n"); - var includedStatuses = sortedStatuses.filter(function (status) { - if (status.maxAge < MINIMUM_REQUIRED_MAX_AGE && !status.forceInclude) { - // dump("INFO: " + status.name + " NOT ON the preload list\n"); - writeTo(status.name + ": " + errorToString(status) + "\n", eos); - return false; - } - - // dump("INFO: " + status.name + " ON the preload list\n"); - if (status.forceInclude && status.error != ERROR_NONE) { - writeTo(status.name + ": " + errorToString(status) + " (error " - + "ignored - included regardless)\n", eos); - } - return true; - }); - - dump("INFO: Writing statuses to file...\n"); - for (var status of includedStatuses) { - writeEntry(status, fos); - } - writeTo(POSTFIX, fos); - FileUtils.closeSafeFileOutputStream(fos); - FileUtils.closeSafeFileOutputStream(eos); - } - catch (e) { - dump("ERROR: problem writing output to '" + OUTPUT + "': " + e + "\n"); - } -} - -function shouldRetry(response) { - return (response.error != ERROR_NO_HSTS_HEADER && - response.error != ERROR_MAX_AGE_TOO_LOW && - response.error != ERROR_NONE && response.retries > 0); -} - -function getHSTSStatuses(inHosts, outStatuses) { - var expectedOutputLength = inHosts.length; - var tmpOutput = []; - var procCount = 0; - for (var i = 0; i < MAX_CONCURRENT_REQUESTS && inHosts.length > 0; i++) { - let host = inHosts.shift(); - dump("spinning off request to '" + host.name + "' (remaining retries: " + - host.retries + ")\n"); - getHSTSStatus(host, tmpOutput); - } - - while (outStatuses.length != expectedOutputLength) { - procCount++; - if (procCount % 200 == 0) gc(); - waitForAResponse(tmpOutput); - var response = tmpOutput.shift(); - dump("request to '" + response.name + "' finished\n"); - if (shouldRetry(response)) { - inHosts.push(response); - } else { - outStatuses.push(response); - } - - if (inHosts.length > 0) { - let host = inHosts.shift(); - dump("[" + procCount + "] spinning off request to '" + host.name + "' (remaining retries: " + - host.retries + ")\n"); - getHSTSStatus(host, tmpOutput); - } - } -} - -// Since all events are processed on the main thread, and since event -// handlers are not preemptible, there shouldn't be any concurrency issues. -function waitForAResponse(outputList) { - // From - var threadManager = Cc["@mozilla.org/thread-manager;1"] - .getService(Ci.nsIThreadManager); - var mainThread = threadManager.currentThread; - while (outputList.length == 0) { - mainThread.processNextEvent(true); - } -} - -function readCurrentList(filename) { - var currentHosts = {}; - var file = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsILocalFile); - file.initWithPath(filename); - var fis = Cc["@mozilla.org/network/file-input-stream;1"] - .createInstance(Ci.nsILineInputStream); - fis.init(file, -1, -1, Ci.nsIFileInputStream.CLOSE_ON_EOF); - var line = {}; - var entryRegex = / { "([^"]*)", (true|false) },/; - while (fis.readLine(line)) { - var match = entryRegex.exec(line.value); - if (match) { - currentHosts[match[1]] = (match[2] == "true"); - } - } - return currentHosts; -} - -function combineLists(newHosts, currentHosts) { - let newHostsSet = new Set(); - - for (let newHost of newHosts) { - newHostsSet.add(newHost.name); - } - - for (let currentHost in currentHosts) { - if (!newHostsSet.has(currentHost)) { - newHosts.push({ name: currentHost, retries: MAX_RETRIES }); - } - } -} - -const TEST_ENTRIES = [ - { name: "includesubdomains.preloaded.test", includeSubdomains: true }, - { name: "includesubdomains2.preloaded.test", includeSubdomains: true }, - { name: "noincludesubdomains.preloaded.test", includeSubdomains: false }, -]; - -function deleteTestHosts(currentHosts) { - for (let testEntry of TEST_ENTRIES) { - delete currentHosts[testEntry.name]; - } -} - -function insertTestHosts(hstsStatuses) { - for (let testEntry of TEST_ENTRIES) { - hstsStatuses.push({ - name: testEntry.name, - maxAge: MINIMUM_REQUIRED_MAX_AGE, - includeSubdomains: testEntry.includeSubdomains, - error: ERROR_NONE, - // This deliberately doesn't have a value for `retries` (because we should - // never attempt to connect to this host). - forceInclude: true, - originalIncludeSubdomains: testEntry.includeSubdomains, - }); - } -} - -// **************************************************************************** -// This is where the action happens: -if (arguments.length != 1) { - throw new Error("Usage: getHSTSPreloadList.js " + - ""); -} -// get the current preload list -var currentHosts = readCurrentList(arguments[0]); -// delete any hosts we use in tests so we don't actually connect to them -deleteTestHosts(currentHosts); -// disable the current preload list so it won't interfere with requests we make -Services.prefs.setBoolPref("network.stricttransportsecurity.preloadlist", false); -// download and parse the raw json file from the Chromium source -var rawdata = download(); -// get just the hosts with mode: "force-https" -var hosts = getHosts(rawdata); -// add hosts in the current list to the new list (avoiding duplicates) -combineLists(hosts, currentHosts); -// get the HSTS status of each host -var hstsStatuses = []; -getHSTSStatuses(hosts, hstsStatuses); -// add the hosts we use in tests -insertTestHosts(hstsStatuses); -// sort the hosts alphabetically -hstsStatuses.sort(compareHSTSStatus); -// write the results to a file (this is where we filter out hosts that we -// either couldn't connect to, didn't receive an HSTS header from, couldn't -// parse the header, or had a header with too short a max-age) -output(hstsStatuses, currentHosts); -// **************************************************************************** -- cgit v1.2.3