Merge branch 'master' of https://github.com/MoonchildProductions/UXP into pm_url_1

4 files changed, 19 insertions, 249 deletions

diff --git a/security/nss/automation/taskcluster/graph/src/context_hash.js b/security/nss/automation/taskcluster/graph/src/context_hash.js
index 0699a0590..f0a2e9a88 100644
--- a/security/nss/automation/taskcluster/graph/src/context_hash.js
+++ b/security/nss/automation/taskcluster/graph/src/context_hash.js
@@ -27,24 +27,14 @@ function collectFilesInDirectory(dir) {
   });
 }
 
-// A list of hashes for each file in the given path.
-function collectFileHashes(context_path) {
+// Compute a context hash for the given context path.
+export default function (context_path) {
   let root = path.join(__dirname, "../../../..");
   let dir = path.join(root, context_path);
   let files = collectFilesInDirectory(dir).sort();
-
-  return files.map(file => {
+  let hashes = files.map(file => {
     return sha256(file + "|" + fs.readFileSync(file, "utf-8"));
   });
-}
-
-// Compute a context hash for the given context path.
-export default function (context_path) {
-  // Regenerate all images when the image_builder changes.
-  let hashes = collectFileHashes("automation/taskcluster/image_builder");
-
-  // Regenerate images when the image itself changes.
-  hashes = hashes.concat(collectFileHashes(context_path));
 
   // Generate a new prefix every month to ensure the image stays buildable.
   let now = new Date();
diff --git a/security/nss/automation/taskcluster/graph/src/extend.js b/security/nss/automation/taskcluster/graph/src/extend.js
index 90e23ae60..d541a1a3b 100644
--- a/security/nss/automation/taskcluster/graph/src/extend.js
+++ b/security/nss/automation/taskcluster/graph/src/extend.js
@@ -15,29 +15,15 @@ const LINUX_CLANG39_IMAGE = {
   path: "automation/taskcluster/docker-clang-3.9"
 };
 
-const LINUX_GCC44_IMAGE = {
-  name: "linux-gcc-4.4",
-  path: "automation/taskcluster/docker-gcc-4.4"
-};
-
 const FUZZ_IMAGE = {
   name: "fuzz",
   path: "automation/taskcluster/docker-fuzz"
 };
 
-const HACL_GEN_IMAGE = {
-  name: "hacl",
-  path: "automation/taskcluster/docker-hacl"
-};
-
 const WINDOWS_CHECKOUT_CMD =
   "bash -c \"hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss || " +
     "(sleep 2; hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss) || " +
     "(sleep 5; hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss)\"";
-const MAC_CHECKOUT_CMD = ["bash", "-c",
-            "hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss || " +
-            "(sleep 2; hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss) || " +
-            "(sleep 5; hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss)"];
 
 /*****************************************************************************/
 
@@ -65,15 +51,6 @@ queue.filter(task => {
     if (task.platform == "aarch64") {
       return false;
     }
-
-    // No mac
-    if (task.platform == "mac") {
-      return false;
-    }
-  }
-
-  if (task.tests == "fips" && task.platform == "mac") {
-    return false;
   }
 
   // Only old make builds have -Ddisable_libpkix=0 and can run chain tests.
@@ -82,8 +59,8 @@ queue.filter(task => {
   }
 
   if (task.group == "Test") {
-    // Don't run test builds on old make platforms, and not for fips gyp.
-    if (task.collection == "make" || task.collection == "fips") {
+    // Don't run test builds on old make platforms
+    if (task.collection == "make") {
       return false;
     }
   }
@@ -101,19 +78,11 @@ queue.filter(task => {
 queue.map(task => {
   if (task.collection == "asan") {
     // CRMF and FIPS tests still leak, unfortunately.
-    if (task.tests == "crmf") {
+    if (task.tests == "crmf" || task.tests == "fips") {
       task.env.ASAN_OPTIONS = "detect_leaks=0";
     }
   }
 
-  // We don't run FIPS SSL tests
-  if (task.tests == "ssl") {
-    if (!task.env) {
-      task.env = {};
-    }
-    task.env.NSS_SSL_TESTS = "crl iopr policy";
-  }
-
   // Windows is slow.
   if (task.platform == "windows2012-64" && task.tests == "chains") {
     task.maxRunTime = 7200;
@@ -159,18 +128,6 @@ export default async function main() {
     ],
   });
 
-  await scheduleLinux("Linux 64 (opt, make)", {
-    env: {USE_64: "1", BUILD_OPT: "1"},
-    platform: "linux64",
-    image: LINUX_IMAGE,
-    collection: "make",
-    command: [
-       "/bin/bash",
-       "-c",
-       "bin/checkout.sh && nss/automation/taskcluster/scripts/build.sh"
-    ],
-  });
-
   await scheduleLinux("Linux 32 (debug, make)", {
     platform: "linux32",
     image: LINUX_IMAGE,
@@ -196,12 +153,6 @@ export default async function main() {
     features: ["allowPtrace"],
   }, "--ubsan --asan");
 
-  await scheduleLinux("Linux 64 (FIPS opt)", {
-    platform: "linux64",
-    collection: "fips",
-    image: LINUX_IMAGE,
-  }, "--enable-fips --opt");
-
   await scheduleWindows("Windows 2012 64 (debug, make)", {
     platform: "windows2012-64",
     collection: "make",
@@ -265,70 +216,6 @@ export default async function main() {
       collection: "opt",
     }, aarch64_base)
   );
-
-  await scheduleMac("Mac (opt)", {collection: "opt"}, "--opt");
-  await scheduleMac("Mac (debug)", {collection: "debug"});
-}
-
-
-async function scheduleMac(name, base, args = "") {
-  let mac_base = merge(base, {
-    env: {
-      PATH: "/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin",
-      NSS_TASKCLUSTER_MAC: "1",
-      DOMSUF: "localdomain",
-      HOST: "localhost",
-    },
-    provisioner: "localprovisioner",
-    workerType: "nss-macos-10-12",
-    platform: "mac"
-  });
-
-  // Build base definition.
-  let build_base = merge({
-    command: [
-      MAC_CHECKOUT_CMD,
-      ["bash", "-c",
-       "nss/automation/taskcluster/scripts/build_gyp.sh", args]
-    ],
-    provisioner: "localprovisioner",
-    workerType: "nss-macos-10-12",
-    platform: "mac",
-    maxRunTime: 7200,
-    artifacts: [{
-      expires: 24 * 7,
-      type: "directory",
-      path: "public"
-    }],
-    kind: "build",
-    symbol: "B"
-  }, mac_base);
-
-  // The task that builds NSPR+NSS.
-  let task_build = queue.scheduleTask(merge(build_base, {name}));
-
-  // The task that generates certificates.
-  let task_cert = queue.scheduleTask(merge(build_base, {
-    name: "Certificates",
-    command: [
-      MAC_CHECKOUT_CMD,
-      ["bash", "-c",
-       "nss/automation/taskcluster/scripts/gen_certs.sh"]
-    ],
-    parent: task_build,
-    symbol: "Certs"
-  }));
-
-  // Schedule tests.
-  scheduleTests(task_build, task_cert, merge(mac_base, {
-    command: [
-      MAC_CHECKOUT_CMD,
-      ["bash", "-c",
-       "nss/automation/taskcluster/scripts/run_tests.sh"]
-    ]
-  }));
-
-  return queue.submit();
 }
 
 /*****************************************************************************/
@@ -355,45 +242,6 @@ async function scheduleLinux(name, base, args = "") {
   // The task that builds NSPR+NSS.
   let task_build = queue.scheduleTask(merge(build_base, {name}));
 
-  // Make builds run FIPS tests, which need an extra FIPS build.
-  if (base.collection == "make") {
-    let extra_build = queue.scheduleTask(merge(build_base, {
-      env: { NSS_FORCE_FIPS: "1" },
-      group: "FIPS",
-      name: `${name} w/ NSS_FORCE_FIPS`
-    }));
-
-    // The task that generates certificates.
-    let task_cert = queue.scheduleTask(merge(build_base, {
-      name: "Certificates",
-      command: [
-        "/bin/bash",
-        "-c",
-        "bin/checkout.sh && nss/automation/taskcluster/scripts/gen_certs.sh"
-      ],
-      parent: extra_build,
-      symbol: "Certs-F",
-      group: "FIPS",
-    }));
-
-    // Schedule FIPS tests.
-    queue.scheduleTask(merge(base, {
-      parent: task_cert,
-      name: "FIPS",
-      command: [
-        "/bin/bash",
-        "-c",
-        "bin/checkout.sh && nss/automation/taskcluster/scripts/run_tests.sh"
-      ],
-      cycle: "standard",
-      kind: "test",
-      name: "FIPS tests",
-      symbol: "Tests-F",
-      tests: "fips",
-      group: "FIPS"
-    }));
-  }
-
   // The task that generates certificates.
   let task_cert = queue.scheduleTask(merge(build_base, {
     name: "Certificates",
@@ -427,26 +275,6 @@ async function scheduleLinux(name, base, args = "") {
   }));
 
   queue.scheduleTask(merge(extra_base, {
-    name: `${name} w/ gcc-4.4`,
-    image: LINUX_GCC44_IMAGE,
-    env: {
-      USE_64: "1",
-      CC: "gcc-4.4",
-      CCC: "g++-4.4",
-      // gcc-4.6 introduced nullptr.
-      NSS_DISABLE_GTESTS: "1",
-    },
-    // Use the old Makefile-based build system, GYP doesn't have a proper GCC
-    // version check for __int128 support. It's mainly meant to cover RHEL6.
-    command: [
-      "/bin/bash",
-      "-c",
-      "bin/checkout.sh && nss/automation/taskcluster/scripts/build.sh",
-    ],
-    symbol: "gcc-4.4"
-  }));
-
-  queue.scheduleTask(merge(extra_base, {
     name: `${name} w/ gcc-4.8`,
     env: {
       CC: "gcc-4.8",
@@ -575,13 +403,12 @@ async function scheduleFuzzing() {
 
   // Schedule MPI fuzzing runs.
   let mpi_base = merge(run_base, {group: "MPI"});
-  let mpi_names = ["add", "addmod", "div", "mod", "mulmod", "sqr",
+  let mpi_names = ["add", "addmod", "div", "expmod", "mod", "mulmod", "sqr",
                    "sqrmod", "sub", "submod"];
   for (let name of mpi_names) {
     scheduleFuzzingRun(mpi_base, `MPI (${name})`, `mpi-${name}`, 4096, name);
   }
   scheduleFuzzingRun(mpi_base, `MPI (invmod)`, `mpi-invmod`, 256, "invmod");
-  scheduleFuzzingRun(mpi_base, `MPI (expmod)`, `mpi-expmod`, 2048, "expmod");
 
   // Schedule TLS fuzzing runs (non-fuzzing mode).
   let tls_base = merge(run_base, {group: "TLS"});
@@ -798,43 +625,6 @@ async function scheduleWindows(name, base, build_script) {
     symbol: "B"
   });
 
-  // Make builds run FIPS tests, which need an extra FIPS build.
-  if (base.collection == "make") {
-    let extra_build = queue.scheduleTask(merge(build_base, {
-      env: { NSS_FORCE_FIPS: "1" },
-      group: "FIPS",
-      name: `${name} w/ NSS_FORCE_FIPS`
-    }));
-
-    // The task that generates certificates.
-    let task_cert = queue.scheduleTask(merge(build_base, {
-      name: "Certificates",
-      command: [
-        WINDOWS_CHECKOUT_CMD,
-        "bash -c nss/automation/taskcluster/windows/gen_certs.sh"
-      ],
-      parent: extra_build,
-      symbol: "Certs-F",
-      group: "FIPS",
-    }));
-
-    // Schedule FIPS tests.
-    queue.scheduleTask(merge(base, {
-      parent: task_cert,
-      name: "FIPS",
-      command: [
-        WINDOWS_CHECKOUT_CMD,
-        "bash -c nss/automation/taskcluster/windows/run_tests.sh"
-      ],
-      cycle: "standard",
-      kind: "test",
-      name: "FIPS tests",
-      symbol: "Tests-F",
-      tests: "fips",
-      group: "FIPS"
-    }));
-  }
-
   // The task that builds NSPR+NSS.
   let task_build = queue.scheduleTask(merge(build_base, {name}));
 
@@ -913,6 +703,9 @@ function scheduleTests(task_build, task_cert, test_base) {
     name: "DB tests", symbol: "DB", tests: "dbtests"
   }));
   queue.scheduleTask(merge(cert_base, {
+    name: "FIPS tests", symbol: "FIPS", tests: "fips"
+  }));
+  queue.scheduleTask(merge(cert_base, {
     name: "Merge tests", symbol: "Merge", tests: "merge"
   }));
   queue.scheduleTask(merge(cert_base, {
@@ -980,16 +773,5 @@ async function scheduleTools() {
     ]
   }));
 
-  queue.scheduleTask(merge(base, {
-    symbol: "hacl",
-    name: "hacl",
-    image: HACL_GEN_IMAGE,
-    command: [
-      "/bin/bash",
-      "-c",
-      "bin/checkout.sh && nss/automation/taskcluster/scripts/run_hacl.sh"
-    ]
-  }));
-
   return queue.submit();
 }
diff --git a/security/nss/automation/taskcluster/graph/src/image_builder.js b/security/nss/automation/taskcluster/graph/src/image_builder.js
index b89b6980c..bc90e0242 100644
--- a/security/nss/automation/taskcluster/graph/src/image_builder.js
+++ b/security/nss/automation/taskcluster/graph/src/image_builder.js
@@ -31,11 +31,13 @@ export async function buildTask({name, path}) {
 
   return {
     name: "Image Builder",
-    image: "nssdev/image_builder:0.1.5",
+    image: "taskcluster/image_builder:0.1.5",
     routes: ["index." + ns],
     env: {
-      NSS_HEAD_REPOSITORY: process.env.NSS_HEAD_REPOSITORY,
-      NSS_HEAD_REVISION: process.env.NSS_HEAD_REVISION,
+      HEAD_REPOSITORY: process.env.NSS_HEAD_REPOSITORY,
+      BASE_REPOSITORY: process.env.NSS_HEAD_REPOSITORY,
+      HEAD_REV: process.env.NSS_HEAD_REVISION,
+      HEAD_REF: process.env.NSS_HEAD_REVISION,
       PROJECT: process.env.TC_PROJECT,
       CONTEXT_PATH: path,
       HASH: hash
@@ -50,11 +52,10 @@ export async function buildTask({name, path}) {
     command: [
       "/bin/bash",
       "-c",
-      "bin/checkout.sh && nss/automation/taskcluster/scripts/build_image.sh"
+      "/home/worker/bin/build_image.sh"
     ],
     platform: "nss-decision",
     features: ["dind"],
-    maxRunTime: 7200,
     kind: "build",
     symbol: "I"
   };
diff --git a/security/nss/automation/taskcluster/graph/src/try_syntax.js b/security/nss/automation/taskcluster/graph/src/try_syntax.js
index 1f4e12eee..7748e068a 100644
--- a/security/nss/automation/taskcluster/graph/src/try_syntax.js
+++ b/security/nss/automation/taskcluster/graph/src/try_syntax.js
@@ -22,10 +22,10 @@ function parseOptions(opts) {
   }
 
   // Parse platforms.
-  let allPlatforms = ["linux", "linux64", "linux64-asan", "linux64-fips",
+  let allPlatforms = ["linux", "linux64", "linux64-asan",
                       "win", "win64", "win-make", "win64-make",
                       "linux64-make", "linux-make", "linux-fuzz",
-                      "linux64-fuzz", "aarch64", "mac"];
+                      "linux64-fuzz", "aarch64"];
   let platforms = intersect(opts.platform.split(/\s*,\s*/), allPlatforms);
 
   // If the given value is nonsense or "none" default to all platforms.
@@ -51,7 +51,7 @@ function parseOptions(opts) {
   }
 
   // Parse tools.
-  let allTools = ["clang-format", "scan-build", "hacl"];
+  let allTools = ["clang-format", "scan-build"];
   let tools = intersect(opts.tools.split(/\s*,\s*/), allTools);
 
   // If the given value is "all" run all tools.
@@ -111,7 +111,6 @@ function filter(opts) {
         "linux": "linux32",
         "linux-fuzz": "linux32",
         "linux64-asan": "linux64",
-        "linux64-fips": "linux64",
         "linux64-fuzz": "linux64",
         "linux64-make": "linux64",
         "linux-make": "linux32",
@@ -127,8 +126,6 @@ function filter(opts) {
       // Additional checks.
       if (platform == "linux64-asan") {
         keep &= coll("asan");
-      } else if (platform == "linux64-fips") {
-        keep &= coll("fips");
       } else if (platform == "linux64-make" || platform == "linux-make" ||
                  platform == "win64-make" || platform == "win-make") {
         keep &= coll("make");