1 files changed, 497 insertions, 0 deletions
diff --git a/security/manager/tools/getHSTSPreloadList.js b/security/manager/tools/getHSTSPreloadList.js
new file mode 100644
index 000000000..5b950f938
--- /dev/null
+++ b/security/manager/tools/getHSTSPreloadList.js
@@ -0,0 +1,497 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+"use strict";
+
+// How to run this file:
+// 1. [obtain firefox source code]
+// 2. [build/obtain firefox binaries]
+// 3. run `[path to]/run-mozilla.sh [path to]/xpcshell \
+//                                  [path to]/getHSTSPreloadlist.js \
+//                                  [absolute path to]/nsSTSPreloadlist.inc'
+// Note: Running this file outputs a new nsSTSPreloadlist.inc in the current
+//       working directory.
+
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cu = Components.utils;
+var Cr = Components.results;
+
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://gre/modules/FileUtils.jsm");
+Cu.import("resource:///modules/XPCOMUtils.jsm");
+
+const SOURCE = "https://chromium.googlesource.com/chromium/src/net/+/master/http/transport_security_state_static.json?format=TEXT";
+const OUTPUT = "nsSTSPreloadList.inc";
+const ERROR_OUTPUT = "nsSTSPreloadList.errors";
+const MINIMUM_REQUIRED_MAX_AGE = 60 * 60 * 24 * 7 * 18;
+const MAX_CONCURRENT_REQUESTS = 5;
+const MAX_RETRIES = 3;
+const REQUEST_TIMEOUT = 30 * 1000;
+const ERROR_NONE = "no error";
+const ERROR_CONNECTING_TO_HOST = "could not connect to host";
+const ERROR_NO_HSTS_HEADER = "did not receive HSTS header";
+const ERROR_MAX_AGE_TOO_LOW = "max-age too low: ";
+const HEADER = "/* This Source Code Form is subject to the terms of the Mozilla Public\n" +
+" * License, v. 2.0. If a copy of the MPL was not distributed with this\n" +
+" * file, You can obtain one at http://mozilla.org/MPL/2.0/. */\n" +
+"\n" +
+"/*****************************************************************************/\n" +
+"/* This is an automatically generated file. If you're not                    */\n" +
+"/* nsSiteSecurityService.cpp, you shouldn't be #including it.     */\n" +
+"/*****************************************************************************/\n" +
+"\n" +
+"#include <stdint.h>\n";
+
+function download() {
+  var req = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"]
+            .createInstance(Ci.nsIXMLHttpRequest);
+  req.open("GET", SOURCE, false); // doing the request synchronously
+  try {
+    req.send();
+  }
+  catch (e) {
+    throw new Error(`ERROR: problem downloading '${SOURCE}': ${e}`);
+  }
+
+  if (req.status != 200) {
+    throw new Error("ERROR: problem downloading '" + SOURCE + "': status " +
+                    req.status);
+  }
+
+  var resultDecoded;
+  try {
+    resultDecoded = atob(req.responseText);
+  }
+  catch (e) {
+    throw new Error("ERROR: could not decode data as base64 from '" + SOURCE +
+                    "': " + e);
+  }
+
+  // we have to filter out '//' comments, while not mangling the json
+  var result = resultDecoded.replace(/^(\s*)?\/\/[^\n]*\n/mg, "");
+  var data = null;
+  try {
+    data = JSON.parse(result);
+  }
+  catch (e) {
+    throw new Error(`ERROR: could not parse data from '${SOURCE}': ${e}`);
+  }
+  return data;
+}
+
+function getHosts(rawdata) {
+  var hosts = [];
+
+  if (!rawdata || !rawdata.entries) {
+    throw new Error("ERROR: source data not formatted correctly: 'entries' " +
+                    "not found");
+  }
+
+  for (let entry of rawdata.entries) {
+    if (entry.mode && entry.mode == "force-https") {
+      if (entry.name) {
+        // We trim the entry name here to avoid malformed URI exceptions when we
+        // later try to connect to the domain.
+        entry.name = entry.name.trim();
+        entry.retries = MAX_RETRIES;
+        entry.originalIncludeSubdomains = entry.include_subdomains;
+        hosts.push(entry);
+      } else {
+        throw new Error("ERROR: entry not formatted correctly: no name found");
+      }
+    }
+  }
+
+  return hosts;
+}
+
+var gSSService = Cc["@mozilla.org/ssservice;1"]
+                   .getService(Ci.nsISiteSecurityService);
+
+function processStsHeader(host, header, status, securityInfo) {
+  var maxAge = { value: 0 };
+  var includeSubdomains = { value: false };
+  var error = ERROR_NONE;
+  if (header != null && securityInfo != null) {
+    try {
+      var uri = Services.io.newURI("https://" + host.name, null, null);
+      var sslStatus = securityInfo.QueryInterface(Ci.nsISSLStatusProvider)
+                                  .SSLStatus;
+      gSSService.processHeader(Ci.nsISiteSecurityService.HEADER_HSTS,
+                               uri, header, sslStatus, 0, maxAge,
+                               includeSubdomains);
+    }
+    catch (e) {
+      dump("ERROR: could not process header '" + header + "' from " +
+           host.name + ": " + e + "\n");
+      error = e;
+    }
+  } else if (status == 0) {
+    error = ERROR_CONNECTING_TO_HOST;
+  } else {
+    error = ERROR_NO_HSTS_HEADER;
+  }
+
+  let forceInclude = (host.forceInclude || host.pins == "google");
+
+  if (error == ERROR_NONE && maxAge.value < MINIMUM_REQUIRED_MAX_AGE) {
+    error = ERROR_MAX_AGE_TOO_LOW;
+  }
+
+  return { name: host.name,
+           maxAge: maxAge.value,
+           includeSubdomains: includeSubdomains.value,
+           error: error,
+           retries: host.retries - 1,
+           forceInclude: forceInclude,
+           originalIncludeSubdomains: host.originalIncludeSubdomains };
+}
+
+// RedirectAndAuthStopper prevents redirects and HTTP authentication
+function RedirectAndAuthStopper() {}
+
+RedirectAndAuthStopper.prototype = {
+  // nsIChannelEventSink
+  asyncOnChannelRedirect: function(oldChannel, newChannel, flags, callback) {
+    throw new Error(Cr.NS_ERROR_ENTITY_CHANGED);
+  },
+
+  // nsIAuthPrompt2
+  promptAuth: function(channel, level, authInfo) {
+    return false;
+  },
+
+  asyncPromptAuth: function(channel, callback, context, level, authInfo) {
+    throw new Error(Cr.NS_ERROR_NOT_IMPLEMENTED);
+  },
+
+  getInterface: function(iid) {
+    return this.QueryInterface(iid);
+  },
+
+  QueryInterface: XPCOMUtils.generateQI([Ci.nsIChannelEventSink,
+                                         Ci.nsIAuthPrompt2])
+};
+
+function getHSTSStatus(host, resultList) {
+  var req = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"]
+            .createInstance(Ci.nsIXMLHttpRequest);
+  var inResultList = false;
+  var uri = "https://" + host.name + "/";
+  req.open("GET", uri, true);
+  req.timeout = REQUEST_TIMEOUT;
+
+  let errorhandler = (evt) => {
+    dump(`ERROR: error making request to ${host.name} (type=${evt.type})\n`);
+    if (!inResultList) {
+      inResultList = true;
+      resultList.push(processStsHeader(host, null, req.status,
+                                       req.channel.securityInfo));
+    }
+  };
+  req.onerror = errorhandler;
+  req.ontimeout = errorhandler;
+  req.onabort = errorhandler;
+
+  req.onload = function(event) {
+    if (!inResultList) {
+      inResultList = true;
+      var header = req.getResponseHeader("strict-transport-security");
+      resultList.push(processStsHeader(host, header, req.status,
+                                       req.channel.securityInfo));
+    }
+  };
+
+  try {
+    req.channel.notificationCallbacks = new RedirectAndAuthStopper();
+    req.send();
+  }
+  catch (e) {
+    dump("ERROR: exception making request to " + host.name + ": " + e + "\n");
+  }
+}
+
+function compareHSTSStatus(a, b) {
+  if (a.name > b.name) {
+    return 1;
+  }
+  if (a.name < b.name) {
+    return -1;
+  }
+  return 0;
+}
+
+function writeTo(string, fos) {
+  fos.write(string, string.length);
+}
+
+// Determines and returns a string representing a declaration of when this
+// preload list should no longer be used.
+// This is the current time plus MINIMUM_REQUIRED_MAX_AGE.
+function getExpirationTimeString() {
+  var now = new Date();
+  var nowMillis = now.getTime();
+  // MINIMUM_REQUIRED_MAX_AGE is in seconds, so convert to milliseconds
+  var expirationMillis = nowMillis + (MINIMUM_REQUIRED_MAX_AGE * 1000);
+  var expirationMicros = expirationMillis * 1000;
+  return "const PRTime gPreloadListExpirationTime = INT64_C(" + expirationMicros + ");\n";
+}
+
+function errorToString(status) {
+  return (status.error == ERROR_MAX_AGE_TOO_LOW
+          ? status.error + status.maxAge
+          : status.error);
+}
+
+function writeEntry(status, indices, outputStream) {
+  let includeSubdomains = (status.finalIncludeSubdomains ? "true" : "false");
+  writeTo("  { " + indices[status.name] + ", " + includeSubdomains + " },\n",
+          outputStream);
+}
+
+function output(sortedStatuses, currentList) {
+  try {
+    var file = FileUtils.getFile("CurWorkD", [OUTPUT]);
+    var errorFile = FileUtils.getFile("CurWorkD", [ERROR_OUTPUT]);
+    var fos = FileUtils.openSafeFileOutputStream(file);
+    var eos = FileUtils.openSafeFileOutputStream(errorFile);
+    writeTo(HEADER, fos);
+    writeTo(getExpirationTimeString(), fos);
+
+    for (let status in sortedStatuses) {
+      // If we've encountered an error for this entry (other than the site not
+      // sending an HSTS header), be safe and don't remove it from the list
+      // (given that it was already on the list).
+      if (status.error != ERROR_NONE &&
+          status.error != ERROR_NO_HSTS_HEADER &&
+          status.error != ERROR_MAX_AGE_TOO_LOW &&
+          status.name in currentList) {
+        dump("INFO: error connecting to or processing " + status.name + " - using previous status on list\n");
+        writeTo(status.name + ": " + errorToString(status) + "\n", eos);
+        status.maxAge = MINIMUM_REQUIRED_MAX_AGE;
+        status.includeSubdomains = currentList[status.name];
+      }
+    }
+
+    // Filter out entries we aren't including.
+    var includedStatuses = sortedStatuses.filter(function (status) {
+      if (status.maxAge < MINIMUM_REQUIRED_MAX_AGE && !status.forceInclude) {
+        dump("INFO: " + status.name + " NOT ON the preload list\n");
+        writeTo(status.name + ": " + errorToString(status) + "\n", eos);
+        return false;
+      }
+
+      dump("INFO: " + status.name + " ON the preload list\n");
+      if (status.forceInclude && status.error != ERROR_NONE) {
+        writeTo(status.name + ": " + errorToString(status) + " (error "
+                + "ignored - included regardless)\n", eos);
+      }
+      return true;
+    });
+
+    // Resolve whether we should include subdomains for each entry.  We could
+    // do this while writing out entries, but separating out that decision is
+    // clearer.  Making that decision here also means we can write the choices
+    // in the comments in the static string table, which makes parsing the
+    // current list significantly easier when we go to update the list.
+    for (let status of includedStatuses) {
+      let incSubdomainsBool = (status.forceInclude && status.error != ERROR_NONE
+                               ? status.originalIncludeSubdomains
+                               : status.includeSubdomains);
+      status.finalIncludeSubdomains = incSubdomainsBool;
+    }
+
+    writeTo("\nstatic const char kSTSHostTable[] = {\n", fos);
+    var indices = {};
+    var currentIndex = 0;
+    for (let status of includedStatuses) {
+      indices[status.name] = currentIndex;
+      // Add 1 for the null terminator in C.
+      currentIndex += status.name.length + 1;
+      // Rebuilding the preload list requires reading the previous preload
+      // list.  Write out a comment describing each host prior to writing out
+      // the string for the host.
+      writeTo("  /* \"" + status.name + "\", " +
+              (status.finalIncludeSubdomains ? "true" : "false") + " */ ",
+              fos);
+      // Write out the string itself as individual characters, including the
+      // null terminator.  We do it this way rather than using C's string
+      // concatentation because some compilers have hardcoded limits on the
+      // lengths of string literals, and the preload list is large enough
+      // that it runs into said limits.
+      for (let c of status.name) {
+	writeTo("'" + c + "', ", fos);
+      }
+      writeTo("'\\0',\n", fos);
+    }
+    writeTo("};\n", fos);
+
+    const PREFIX = "\n" +
+      "struct nsSTSPreload\n" +
+      "{\n" +
+      "  const uint32_t mHostIndex : 31;\n" +
+      "  const uint32_t mIncludeSubdomains : 1;\n" +
+      "};\n" +
+      "\n" +
+      "static const nsSTSPreload kSTSPreloadList[] = {\n";
+    const POSTFIX = "};\n";
+
+    writeTo(PREFIX, fos);
+    for (let status of includedStatuses) {
+      writeEntry(status, indices, fos);
+    }
+    writeTo(POSTFIX, fos);
+    FileUtils.closeSafeFileOutputStream(fos);
+    FileUtils.closeSafeFileOutputStream(eos);
+  }
+  catch (e) {
+    dump("ERROR: problem writing output to '" + OUTPUT + "': " + e + "\n");
+  }
+}
+
+function shouldRetry(response) {
+  return (response.error != ERROR_NO_HSTS_HEADER &&
+          response.error != ERROR_MAX_AGE_TOO_LOW &&
+          response.error != ERROR_NONE && response.retries > 0);
+}
+
+function getHSTSStatuses(inHosts, outStatuses) {
+  var expectedOutputLength = inHosts.length;
+  var tmpOutput = [];
+  for (var i = 0; i < MAX_CONCURRENT_REQUESTS && inHosts.length > 0; i++) {
+    let host = inHosts.shift();
+    dump("spinning off request to '" + host.name + "' (remaining retries: " +
+         host.retries + ")\n");
+    getHSTSStatus(host, tmpOutput);
+  }
+
+  while (outStatuses.length != expectedOutputLength) {
+    waitForAResponse(tmpOutput);
+    var response = tmpOutput.shift();
+    dump("request to '" + response.name + "' finished\n");
+    if (shouldRetry(response)) {
+      inHosts.push(response);
+    } else {
+      outStatuses.push(response);
+    }
+
+    if (inHosts.length > 0) {
+      let host = inHosts.shift();
+      dump("spinning off request to '" + host.name + "' (remaining retries: " +
+           host.retries + ")\n");
+      getHSTSStatus(host, tmpOutput);
+    }
+  }
+}
+
+// Since all events are processed on the main thread, and since event
+// handlers are not preemptible, there shouldn't be any concurrency issues.
+function waitForAResponse(outputList) {
+  // From <https://developer.mozilla.org/en/XPConnect/xpcshell/HOWTO>
+  var threadManager = Cc["@mozilla.org/thread-manager;1"]
+                      .getService(Ci.nsIThreadManager);
+  var mainThread = threadManager.currentThread;
+  while (outputList.length == 0) {
+    mainThread.processNextEvent(true);
+  }
+}
+
+function readCurrentList(filename) {
+  var currentHosts = {};
+  var file = Cc["@mozilla.org/file/local;1"].createInstance(Ci.nsILocalFile);
+  file.initWithPath(filename);
+  var fis = Cc["@mozilla.org/network/file-input-stream;1"]
+              .createInstance(Ci.nsILineInputStream);
+  fis.init(file, -1, -1, Ci.nsIFileInputStream.CLOSE_ON_EOF);
+  var line = {};
+  // While we generate entries matching the version 2 format (see bug 1255425
+  // for details), we still need to be able to read entries in the version 1
+  // format for bootstrapping a version 2 preload list from a version 1
+  // preload list.  Hence these two regexes.
+  var v1EntryRegex = /  { "([^"]*)", (true|false) },/;
+  var v2EntryRegex = /  \/\* "([^"]*)", (true|false) \*\//;
+  while (fis.readLine(line)) {
+    var match = v1EntryRegex.exec(line.value);
+    if (!match) {
+      match = v2EntryRegex.exec(line.value);
+    }
+    if (match) {
+      currentHosts[match[1]] = (match[2] == "true");
+    }
+  }
+  return currentHosts;
+}
+
+function combineLists(newHosts, currentHosts) {
+  for (let currentHost in currentHosts) {
+    let found = false;
+    for (let newHost of newHosts) {
+      if (newHost.name == currentHost) {
+        found = true;
+        break;
+      }
+    }
+    if (!found) {
+      newHosts.push({ name: currentHost, retries: MAX_RETRIES });
+    }
+  }
+}
+
+const TEST_ENTRIES = [
+  { name: "includesubdomains.preloaded.test", includeSubdomains: true },
+  { name: "includesubdomains2.preloaded.test", includeSubdomains: true },
+  { name: "noincludesubdomains.preloaded.test", includeSubdomains: false },
+];
+
+function deleteTestHosts(currentHosts) {
+  for (let testEntry of TEST_ENTRIES) {
+    delete currentHosts[testEntry.name];
+  }
+}
+
+function insertTestHosts(hstsStatuses) {
+  for (let testEntry of TEST_ENTRIES) {
+    hstsStatuses.push({
+      name: testEntry.name,
+      maxAge: MINIMUM_REQUIRED_MAX_AGE,
+      includeSubdomains: testEntry.includeSubdomains,
+      error: ERROR_NONE,
+      // This deliberately doesn't have a value for `retries` (because we should
+      // never attempt to connect to this host).
+      forceInclude: true,
+      originalIncludeSubdomains: testEntry.includeSubdomains,
+    });
+  }
+}
+
+// ****************************************************************************
+// This is where the action happens:
+if (arguments.length != 1) {
+  throw new Error("Usage: getHSTSPreloadList.js " +
+                  "<absolute path to current nsSTSPreloadList.inc>");
+}
+// get the current preload list
+var currentHosts = readCurrentList(arguments[0]);
+// delete any hosts we use in tests so we don't actually connect to them
+deleteTestHosts(currentHosts);
+// disable the current preload list so it won't interfere with requests we make
+Services.prefs.setBoolPref("network.stricttransportsecurity.preloadlist", false);
+// download and parse the raw json file from the Chromium source
+var rawdata = download();
+// get just the hosts with mode: "force-https"
+var hosts = getHosts(rawdata);
+// add hosts in the current list to the new list (avoiding duplicates)
+combineLists(hosts, currentHosts);
+// get the HSTS status of each host
+var hstsStatuses = [];
+getHSTSStatuses(hosts, hstsStatuses);
+// add the hosts we use in tests
+insertTestHosts(hstsStatuses);
+// sort the hosts alphabetically
+hstsStatuses.sort(compareHSTSStatus);
+// write the results to a file (this is where we filter out hosts that we
+// either couldn't connect to, didn't receive an HSTS header from, couldn't
+// parse the header, or had a header with too short a max-age)
+output(hstsStatuses, currentHosts);
+// ****************************************************************************