diff options
Diffstat (limited to 'security/nss/automation/taskcluster/graph')
5 files changed, 367 insertions, 34 deletions
diff --git a/security/nss/automation/taskcluster/graph/src/context_hash.js b/security/nss/automation/taskcluster/graph/src/context_hash.js index f0a2e9a88..0699a0590 100644 --- a/security/nss/automation/taskcluster/graph/src/context_hash.js +++ b/security/nss/automation/taskcluster/graph/src/context_hash.js @@ -27,14 +27,24 @@ function collectFilesInDirectory(dir) { }); } -// Compute a context hash for the given context path. -export default function (context_path) { +// A list of hashes for each file in the given path. +function collectFileHashes(context_path) { let root = path.join(__dirname, "../../../.."); let dir = path.join(root, context_path); let files = collectFilesInDirectory(dir).sort(); - let hashes = files.map(file => { + + return files.map(file => { return sha256(file + "|" + fs.readFileSync(file, "utf-8")); }); +} + +// Compute a context hash for the given context path. +export default function (context_path) { + // Regenerate all images when the image_builder changes. + let hashes = collectFileHashes("automation/taskcluster/image_builder"); + + // Regenerate images when the image itself changes. + hashes = hashes.concat(collectFileHashes(context_path)); // Generate a new prefix every month to ensure the image stays buildable. let now = new Date(); diff --git a/security/nss/automation/taskcluster/graph/src/extend.js b/security/nss/automation/taskcluster/graph/src/extend.js index d541a1a3b..ee9ac9b74 100644 --- a/security/nss/automation/taskcluster/graph/src/extend.js +++ b/security/nss/automation/taskcluster/graph/src/extend.js @@ -15,15 +15,34 @@ const LINUX_CLANG39_IMAGE = { path: "automation/taskcluster/docker-clang-3.9" }; +const LINUX_GCC44_IMAGE = { + name: "linux-gcc-4.4", + path: "automation/taskcluster/docker-gcc-4.4" +}; + const FUZZ_IMAGE = { name: "fuzz", path: "automation/taskcluster/docker-fuzz" }; +const HACL_GEN_IMAGE = { + name: "hacl", + path: "automation/taskcluster/docker-hacl" +}; + +const SAW_IMAGE = { + name: "saw", + path: "automation/taskcluster/docker-saw" +}; + const WINDOWS_CHECKOUT_CMD = "bash -c \"hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss || " + "(sleep 2; hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss) || " + "(sleep 5; hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss)\""; +const MAC_CHECKOUT_CMD = ["bash", "-c", + "hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss || " + + "(sleep 2; hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss) || " + + "(sleep 5; hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss)"]; /*****************************************************************************/ @@ -51,6 +70,16 @@ queue.filter(task => { if (task.platform == "aarch64") { return false; } + + // No mac + if (task.platform == "mac") { + return false; + } + } + + if (task.tests == "fips" && + (task.platform == "mac" || task.platform == "aarch64")) { + return false; } // Only old make builds have -Ddisable_libpkix=0 and can run chain tests. @@ -59,13 +88,13 @@ queue.filter(task => { } if (task.group == "Test") { - // Don't run test builds on old make platforms - if (task.collection == "make") { + // Don't run test builds on old make platforms, and not for fips gyp. + if (task.collection == "make" || task.collection == "fips") { return false; } } - // Don't run additional hardware tests on ARM (we don't have anything there). + // Don't run all additional hardware tests on ARM. if (task.group == "Cipher" && task.platform == "aarch64" && task.env && (task.env.NSS_DISABLE_PCLMUL == "1" || task.env.NSS_DISABLE_HW_AES == "1" || task.env.NSS_DISABLE_AVX == "1")) { @@ -78,11 +107,19 @@ queue.filter(task => { queue.map(task => { if (task.collection == "asan") { // CRMF and FIPS tests still leak, unfortunately. - if (task.tests == "crmf" || task.tests == "fips") { + if (task.tests == "crmf") { task.env.ASAN_OPTIONS = "detect_leaks=0"; } } + // We don't run FIPS SSL tests + if (task.tests == "ssl") { + if (!task.env) { + task.env = {}; + } + task.env.NSS_SSL_TESTS = "crl iopr policy"; + } + // Windows is slow. if (task.platform == "windows2012-64" && task.tests == "chains") { task.maxRunTime = 7200; @@ -128,6 +165,18 @@ export default async function main() { ], }); + await scheduleLinux("Linux 64 (opt, make)", { + env: {USE_64: "1", BUILD_OPT: "1"}, + platform: "linux64", + image: LINUX_IMAGE, + collection: "make", + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/build.sh" + ], + }); + await scheduleLinux("Linux 32 (debug, make)", { platform: "linux32", image: LINUX_IMAGE, @@ -144,8 +193,8 @@ export default async function main() { UBSAN_OPTIONS: "print_stacktrace=1", NSS_DISABLE_ARENA_FREE_LIST: "1", NSS_DISABLE_UNLOAD: "1", - CC: "clang", - CCC: "clang++", + CC: "clang-5.0", + CCC: "clang++-5.0", }, platform: "linux64", collection: "asan", @@ -153,6 +202,12 @@ export default async function main() { features: ["allowPtrace"], }, "--ubsan --asan"); + await scheduleLinux("Linux 64 (FIPS opt)", { + platform: "linux64", + collection: "fips", + image: LINUX_IMAGE, + }, "--enable-fips --opt"); + await scheduleWindows("Windows 2012 64 (debug, make)", { platform: "windows2012-64", collection: "make", @@ -216,6 +271,82 @@ export default async function main() { collection: "opt", }, aarch64_base) ); + + await scheduleLinux("Linux AArch64 (debug, make)", + merge({ + env: {USE_64: "1"}, + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/build.sh" + ], + collection: "make", + }, aarch64_base) + ); + + await scheduleMac("Mac (opt)", {collection: "opt"}, "--opt"); + await scheduleMac("Mac (debug)", {collection: "debug"}); +} + + +async function scheduleMac(name, base, args = "") { + let mac_base = merge(base, { + env: { + PATH: "/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin", + NSS_TASKCLUSTER_MAC: "1", + DOMSUF: "localdomain", + HOST: "localhost", + }, + provisioner: "localprovisioner", + workerType: "nss-macos-10-12", + platform: "mac" + }); + + // Build base definition. + let build_base = merge({ + command: [ + MAC_CHECKOUT_CMD, + ["bash", "-c", + "nss/automation/taskcluster/scripts/build_gyp.sh", args] + ], + provisioner: "localprovisioner", + workerType: "nss-macos-10-12", + platform: "mac", + maxRunTime: 7200, + artifacts: [{ + expires: 24 * 7, + type: "directory", + path: "public" + }], + kind: "build", + symbol: "B" + }, mac_base); + + // The task that builds NSPR+NSS. + let task_build = queue.scheduleTask(merge(build_base, {name})); + + // The task that generates certificates. + let task_cert = queue.scheduleTask(merge(build_base, { + name: "Certificates", + command: [ + MAC_CHECKOUT_CMD, + ["bash", "-c", + "nss/automation/taskcluster/scripts/gen_certs.sh"] + ], + parent: task_build, + symbol: "Certs" + })); + + // Schedule tests. + scheduleTests(task_build, task_cert, merge(mac_base, { + command: [ + MAC_CHECKOUT_CMD, + ["bash", "-c", + "nss/automation/taskcluster/scripts/run_tests.sh"] + ] + })); + + return queue.submit(); } /*****************************************************************************/ @@ -242,6 +373,45 @@ async function scheduleLinux(name, base, args = "") { // The task that builds NSPR+NSS. let task_build = queue.scheduleTask(merge(build_base, {name})); + // Make builds run FIPS tests, which need an extra FIPS build. + if (base.collection == "make") { + let extra_build = queue.scheduleTask(merge(build_base, { + env: { NSS_FORCE_FIPS: "1" }, + group: "FIPS", + name: `${name} w/ NSS_FORCE_FIPS` + })); + + // The task that generates certificates. + let task_cert = queue.scheduleTask(merge(build_base, { + name: "Certificates", + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/gen_certs.sh" + ], + parent: extra_build, + symbol: "Certs-F", + group: "FIPS", + })); + + // Schedule FIPS tests. + queue.scheduleTask(merge(base, { + parent: task_cert, + name: "FIPS", + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/run_tests.sh" + ], + cycle: "standard", + kind: "test", + name: "FIPS tests", + symbol: "Tests-F", + tests: "fips", + group: "FIPS" + })); + } + // The task that generates certificates. let task_cert = queue.scheduleTask(merge(build_base, { name: "Certificates", @@ -266,12 +436,32 @@ async function scheduleLinux(name, base, args = "") { // Extra builds. let extra_base = merge({group: "Builds"}, build_base); queue.scheduleTask(merge(extra_base, { - name: `${name} w/ clang-4.0`, + name: `${name} w/ clang-5.0`, env: { - CC: "clang", - CCC: "clang++", + CC: "clang-5.0", + CCC: "clang++-5.0", }, - symbol: "clang-4.0" + symbol: "clang-5.0" + })); + + queue.scheduleTask(merge(extra_base, { + name: `${name} w/ gcc-4.4`, + image: LINUX_GCC44_IMAGE, + env: { + USE_64: "1", + CC: "gcc-4.4", + CCC: "g++-4.4", + // gcc-4.6 introduced nullptr. + NSS_DISABLE_GTESTS: "1", + }, + // Use the old Makefile-based build system, GYP doesn't have a proper GCC + // version check for __int128 support. It's mainly meant to cover RHEL6. + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/build.sh", + ], + symbol: "gcc-4.4" })); queue.scheduleTask(merge(extra_base, { @@ -403,12 +593,13 @@ async function scheduleFuzzing() { // Schedule MPI fuzzing runs. let mpi_base = merge(run_base, {group: "MPI"}); - let mpi_names = ["add", "addmod", "div", "expmod", "mod", "mulmod", "sqr", + let mpi_names = ["add", "addmod", "div", "mod", "mulmod", "sqr", "sqrmod", "sub", "submod"]; for (let name of mpi_names) { scheduleFuzzingRun(mpi_base, `MPI (${name})`, `mpi-${name}`, 4096, name); } scheduleFuzzingRun(mpi_base, `MPI (invmod)`, `mpi-invmod`, 256, "invmod"); + scheduleFuzzingRun(mpi_base, `MPI (expmod)`, `mpi-expmod`, 2048, "expmod"); // Schedule TLS fuzzing runs (non-fuzzing mode). let tls_base = merge(run_base, {group: "TLS"}); @@ -625,6 +816,43 @@ async function scheduleWindows(name, base, build_script) { symbol: "B" }); + // Make builds run FIPS tests, which need an extra FIPS build. + if (base.collection == "make") { + let extra_build = queue.scheduleTask(merge(build_base, { + env: { NSS_FORCE_FIPS: "1" }, + group: "FIPS", + name: `${name} w/ NSS_FORCE_FIPS` + })); + + // The task that generates certificates. + let task_cert = queue.scheduleTask(merge(build_base, { + name: "Certificates", + command: [ + WINDOWS_CHECKOUT_CMD, + "bash -c nss/automation/taskcluster/windows/gen_certs.sh" + ], + parent: extra_build, + symbol: "Certs-F", + group: "FIPS", + })); + + // Schedule FIPS tests. + queue.scheduleTask(merge(base, { + parent: task_cert, + name: "FIPS", + command: [ + WINDOWS_CHECKOUT_CMD, + "bash -c nss/automation/taskcluster/windows/run_tests.sh" + ], + cycle: "standard", + kind: "test", + name: "FIPS tests", + symbol: "Tests-F", + tests: "fips", + group: "FIPS" + })); + } + // The task that builds NSPR+NSS. let task_build = queue.scheduleTask(merge(build_base, {name})); @@ -685,6 +913,13 @@ function scheduleTests(task_build, task_cert, test_base) { env: {NSS_DISABLE_AVX: "1"}, group: "Cipher" })); queue.scheduleTask(merge(no_cert_base, { + name: "Cipher tests", symbol: "NoSSSE3|NEON", tests: "cipher", + env: { + NSS_DISABLE_ARM_NEON: "1", + NSS_DISABLE_SSSE3: "1" + }, group: "Cipher" + })); + queue.scheduleTask(merge(no_cert_base, { name: "EC tests", symbol: "EC", tests: "ec" })); queue.scheduleTask(merge(no_cert_base, { @@ -703,9 +938,6 @@ function scheduleTests(task_build, task_cert, test_base) { name: "DB tests", symbol: "DB", tests: "dbtests" })); queue.scheduleTask(merge(cert_base, { - name: "FIPS tests", symbol: "FIPS", tests: "fips" - })); - queue.scheduleTask(merge(cert_base, { name: "Merge tests", symbol: "Merge", tests: "merge" })); queue.scheduleTask(merge(cert_base, { @@ -739,6 +971,18 @@ async function scheduleTools() { kind: "test" }; + //ABI check task + queue.scheduleTask(merge(base, { + symbol: "abi", + name: "abi", + image: LINUX_IMAGE, + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/check_abi.sh" + ], + })); + queue.scheduleTask(merge(base, { symbol: "clang-format-3.9", name: "clang-format-3.9", @@ -751,13 +995,13 @@ async function scheduleTools() { })); queue.scheduleTask(merge(base, { - symbol: "scan-build-4.0", - name: "scan-build-4.0", + symbol: "scan-build-5.0", + name: "scan-build-5.0", image: LINUX_IMAGE, env: { USE_64: "1", - CC: "clang", - CCC: "clang++", + CC: "clang-5.0", + CCC: "clang++-5.0", }, artifacts: { public: { @@ -773,5 +1017,80 @@ async function scheduleTools() { ] })); + queue.scheduleTask(merge(base, { + symbol: "hacl", + name: "hacl", + image: HACL_GEN_IMAGE, + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/run_hacl.sh" + ] + })); + + let task_saw = queue.scheduleTask(merge(base, { + symbol: "B", + group: "SAW", + name: "LLVM bitcode build (32 bit)", + image: SAW_IMAGE, + kind: "build", + env: { + AR: "llvm-ar-3.8", + CC: "clang-3.8", + CCC: "clang++-3.8" + }, + artifacts: { + public: { + expires: 24 * 7, + type: "directory", + path: "/home/worker/artifacts" + } + }, + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/build_gyp.sh --disable-tests --emit-llvm -m32" + ] + })); + + queue.scheduleTask(merge(base, { + parent: task_saw, + symbol: "bmul", + group: "SAW", + name: "bmul.saw", + image: SAW_IMAGE, + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/run_saw.sh bmul" + ] + })); + + queue.scheduleTask(merge(base, { + parent: task_saw, + symbol: "ChaCha20", + group: "SAW", + name: "chacha20.saw", + image: SAW_IMAGE, + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/run_saw.sh chacha20" + ] + })); + + queue.scheduleTask(merge(base, { + parent: task_saw, + symbol: "Poly1305", + group: "SAW", + name: "poly1305.saw", + image: SAW_IMAGE, + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/run_saw.sh poly1305" + ] + })); + return queue.submit(); } diff --git a/security/nss/automation/taskcluster/graph/src/image_builder.js b/security/nss/automation/taskcluster/graph/src/image_builder.js index bc90e0242..d9d7755dc 100644 --- a/security/nss/automation/taskcluster/graph/src/image_builder.js +++ b/security/nss/automation/taskcluster/graph/src/image_builder.js @@ -30,14 +30,12 @@ export async function buildTask({name, path}) { let ns = `docker.images.v1.${process.env.TC_PROJECT}.${name}.hash.${hash}`; return { - name: "Image Builder", - image: "taskcluster/image_builder:0.1.5", + name: `Image Builder (${name})`, + image: "nssdev/image_builder:0.1.5", routes: ["index." + ns], env: { - HEAD_REPOSITORY: process.env.NSS_HEAD_REPOSITORY, - BASE_REPOSITORY: process.env.NSS_HEAD_REPOSITORY, - HEAD_REV: process.env.NSS_HEAD_REVISION, - HEAD_REF: process.env.NSS_HEAD_REVISION, + NSS_HEAD_REPOSITORY: process.env.NSS_HEAD_REPOSITORY, + NSS_HEAD_REVISION: process.env.NSS_HEAD_REVISION, PROJECT: process.env.TC_PROJECT, CONTEXT_PATH: path, HASH: hash @@ -52,10 +50,11 @@ export async function buildTask({name, path}) { command: [ "/bin/bash", "-c", - "/home/worker/bin/build_image.sh" + "bin/checkout.sh && nss/automation/taskcluster/scripts/build_image.sh" ], platform: "nss-decision", features: ["dind"], + maxRunTime: 7200, kind: "build", symbol: "I" }; diff --git a/security/nss/automation/taskcluster/graph/src/queue.js b/security/nss/automation/taskcluster/graph/src/queue.js index 29b570729..809a17bf1 100644 --- a/security/nss/automation/taskcluster/graph/src/queue.js +++ b/security/nss/automation/taskcluster/graph/src/queue.js @@ -31,10 +31,11 @@ function parseRoutes(routes) { ]; // Notify about failures (except on try). - if (process.env.TC_PROJECT != "nss-try") { + // Turned off, too noisy. + /*if (process.env.TC_PROJECT != "nss-try") { rv.push(`notify.email.${process.env.TC_OWNER}.on-failed`, `notify.email.${process.env.TC_OWNER}.on-exception`); - } + }*/ return rv; } diff --git a/security/nss/automation/taskcluster/graph/src/try_syntax.js b/security/nss/automation/taskcluster/graph/src/try_syntax.js index 7748e068a..1c06dde13 100644 --- a/security/nss/automation/taskcluster/graph/src/try_syntax.js +++ b/security/nss/automation/taskcluster/graph/src/try_syntax.js @@ -22,10 +22,10 @@ function parseOptions(opts) { } // Parse platforms. - let allPlatforms = ["linux", "linux64", "linux64-asan", + let allPlatforms = ["linux", "linux64", "linux64-asan", "linux64-fips", "win", "win64", "win-make", "win64-make", "linux64-make", "linux-make", "linux-fuzz", - "linux64-fuzz", "aarch64"]; + "linux64-fuzz", "aarch64", "mac"]; let platforms = intersect(opts.platform.split(/\s*,\s*/), allPlatforms); // If the given value is nonsense or "none" default to all platforms. @@ -51,7 +51,7 @@ function parseOptions(opts) { } // Parse tools. - let allTools = ["clang-format", "scan-build"]; + let allTools = ["clang-format", "scan-build", "hacl", "saw", "abi"]; let tools = intersect(opts.tools.split(/\s*,\s*/), allTools); // If the given value is "all" run all tools. @@ -77,7 +77,8 @@ function filter(opts) { // are not affected by platform or build type selectors. if (task.platform == "nss-tools") { return opts.tools.some(tool => { - return task.symbol.toLowerCase().startsWith(tool); + return task.symbol.toLowerCase().startsWith(tool) || + (task.group && task.group.toLowerCase().startsWith(tool)); }); } @@ -111,6 +112,7 @@ function filter(opts) { "linux": "linux32", "linux-fuzz": "linux32", "linux64-asan": "linux64", + "linux64-fips": "linux64", "linux64-fuzz": "linux64", "linux64-make": "linux64", "linux-make": "linux32", @@ -126,6 +128,8 @@ function filter(opts) { // Additional checks. if (platform == "linux64-asan") { keep &= coll("asan"); + } else if (platform == "linux64-fips") { + keep &= coll("fips"); } else if (platform == "linux64-make" || platform == "linux-make" || platform == "win64-make" || platform == "win-make") { keep &= coll("make"); |