Merge pull request #461 from trav90/HSTS

Improve HSTS preload list generation

1 files changed, 26 insertions, 69 deletions

diff --git a/security/manager/tools/getHSTSPreloadList.js b/security/manager/tools/getHSTSPreloadList.js
index 5b950f938..518a9246b 100644
--- a/security/manager/tools/getHSTSPreloadList.js
+++ b/security/manager/tools/getHSTSPreloadList.js
@@ -25,7 +25,7 @@ const SOURCE = "https://chromium.googlesource.com/chromium/src/net/+/master/http
 const OUTPUT = "nsSTSPreloadList.inc";
 const ERROR_OUTPUT = "nsSTSPreloadList.errors";
 const MINIMUM_REQUIRED_MAX_AGE = 60 * 60 * 24 * 7 * 18;
-const MAX_CONCURRENT_REQUESTS = 5;
+const MAX_CONCURRENT_REQUESTS = 15;
 const MAX_RETRIES = 3;
 const REQUEST_TIMEOUT = 30 * 1000;
 const ERROR_NONE = "no error";
@@ -42,6 +42,16 @@ const HEADER = "/* This Source Code Form is subject to the terms of the Mozilla
 "/*****************************************************************************/\n" +
 "\n" +
 "#include <stdint.h>\n";
+const PREFIX = "\n" +
+"class nsSTSPreload\n" +
+"{\n" +
+"  public:\n" +
+"    const char *mHost;\n" +
+"    const bool mIncludeSubdomains;\n" +
+"};\n" +
+"\n" +
+"static const nsSTSPreload kSTSPreloadList[] = {\n";
+const POSTFIX =  "};\n";
 
 function download() {
   var req = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"]
@@ -244,9 +254,12 @@ function errorToString(status) {
           : status.error);
 }
 
-function writeEntry(status, indices, outputStream) {
-  let includeSubdomains = (status.finalIncludeSubdomains ? "true" : "false");
-  writeTo("  { " + indices[status.name] + ", " + includeSubdomains + " },\n",
+function writeEntry(status, outputStream) {
+  let incSubdomainsBool = (status.forceInclude && status.error != ERROR_NONE
+                           ? status.originalIncludeSubdomains
+                           : status.includeSubdomains);
+  let includeSubdomains = (incSubdomainsBool ? "true" : "false");
+  writeTo("  { \"" + status.name + "\", " + includeSubdomains + " },\n",
           outputStream);
 }
 
@@ -258,19 +271,19 @@ function output(sortedStatuses, currentList) {
     var eos = FileUtils.openSafeFileOutputStream(errorFile);
     writeTo(HEADER, fos);
     writeTo(getExpirationTimeString(), fos);
+    writeTo(PREFIX, fos);
 
     for (let status in sortedStatuses) {
       // If we've encountered an error for this entry (other than the site not
-      // sending an HSTS header), be safe and don't remove it from the list
-      // (given that it was already on the list).
+      // sending an HSTS header), be safe and remove it from the list
+      // (preventing stale entries from accumulating).
       if (status.error != ERROR_NONE &&
           status.error != ERROR_NO_HSTS_HEADER &&
           status.error != ERROR_MAX_AGE_TOO_LOW &&
           status.name in currentList) {
-        dump("INFO: error connecting to or processing " + status.name + " - using previous status on list\n");
+        dump("INFO: error connecting to or processing " + status.name + " - dropping from list\n");
         writeTo(status.name + ": " + errorToString(status) + "\n", eos);
-        status.maxAge = MINIMUM_REQUIRED_MAX_AGE;
-        status.includeSubdomains = currentList[status.name];
+        status.maxAge = 0;
       }
     }
 
@@ -290,56 +303,8 @@ function output(sortedStatuses, currentList) {
       return true;
     });
 
-    // Resolve whether we should include subdomains for each entry.  We could
-    // do this while writing out entries, but separating out that decision is
-    // clearer.  Making that decision here also means we can write the choices
-    // in the comments in the static string table, which makes parsing the
-    // current list significantly easier when we go to update the list.
-    for (let status of includedStatuses) {
-      let incSubdomainsBool = (status.forceInclude && status.error != ERROR_NONE
-                               ? status.originalIncludeSubdomains
-                               : status.includeSubdomains);
-      status.finalIncludeSubdomains = incSubdomainsBool;
-    }
-
-    writeTo("\nstatic const char kSTSHostTable[] = {\n", fos);
-    var indices = {};
-    var currentIndex = 0;
-    for (let status of includedStatuses) {
-      indices[status.name] = currentIndex;
-      // Add 1 for the null terminator in C.
-      currentIndex += status.name.length + 1;
-      // Rebuilding the preload list requires reading the previous preload
-      // list.  Write out a comment describing each host prior to writing out
-      // the string for the host.
-      writeTo("  /* \"" + status.name + "\", " +
-              (status.finalIncludeSubdomains ? "true" : "false") + " */ ",
-              fos);
-      // Write out the string itself as individual characters, including the
-      // null terminator.  We do it this way rather than using C's string
-      // concatentation because some compilers have hardcoded limits on the
-      // lengths of string literals, and the preload list is large enough
-      // that it runs into said limits.
-      for (let c of status.name) {
-	writeTo("'" + c + "', ", fos);
-      }
-      writeTo("'\\0',\n", fos);
-    }
-    writeTo("};\n", fos);
-
-    const PREFIX = "\n" +
-      "struct nsSTSPreload\n" +
-      "{\n" +
-      "  const uint32_t mHostIndex : 31;\n" +
-      "  const uint32_t mIncludeSubdomains : 1;\n" +
-      "};\n" +
-      "\n" +
-      "static const nsSTSPreload kSTSPreloadList[] = {\n";
-    const POSTFIX = "};\n";
-
-    writeTo(PREFIX, fos);
-    for (let status of includedStatuses) {
-      writeEntry(status, indices, fos);
+    for (var status of includedStatuses) {
+      writeEntry(status, fos);
     }
     writeTo(POSTFIX, fos);
     FileUtils.closeSafeFileOutputStream(fos);
@@ -405,17 +370,9 @@ function readCurrentList(filename) {
               .createInstance(Ci.nsILineInputStream);
   fis.init(file, -1, -1, Ci.nsIFileInputStream.CLOSE_ON_EOF);
   var line = {};
-  // While we generate entries matching the version 2 format (see bug 1255425
-  // for details), we still need to be able to read entries in the version 1
-  // format for bootstrapping a version 2 preload list from a version 1
-  // preload list.  Hence these two regexes.
-  var v1EntryRegex = /  { "([^"]*)", (true|false) },/;
-  var v2EntryRegex = /  \/\* "([^"]*)", (true|false) \*\//;
+  var entryRegex = /  { "([^"]*)", (true|false) },/;
   while (fis.readLine(line)) {
-    var match = v1EntryRegex.exec(line.value);
-    if (!match) {
-      match = v2EntryRegex.exec(line.value);
-    }
+    var match = entryRegex.exec(line.value);
     if (match) {
       currentHosts[match[1]] = (match[2] == "true");
     }