diff options
Diffstat (limited to 'security/nss/automation/taskcluster/graph/src')
7 files changed, 1102 insertions, 0 deletions
diff --git a/security/nss/automation/taskcluster/graph/src/context_hash.js b/security/nss/automation/taskcluster/graph/src/context_hash.js new file mode 100644 index 000000000..f0a2e9a88 --- /dev/null +++ b/security/nss/automation/taskcluster/graph/src/context_hash.js @@ -0,0 +1,43 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import fs from "fs"; +import path from "path"; +import crypto from "crypto"; +import flatmap from "flatmap"; + +// Compute the SHA-256 digest. +function sha256(data) { + let hash = crypto.createHash("sha256"); + hash.update(data); + return hash.digest("hex"); +} + +// Recursively collect a list of all files of a given directory. +function collectFilesInDirectory(dir) { + return flatmap(fs.readdirSync(dir), entry => { + let entry_path = path.join(dir, entry); + + if (fs.lstatSync(entry_path).isDirectory()) { + return collectFilesInDirectory(entry_path); + } + + return [entry_path]; + }); +} + +// Compute a context hash for the given context path. +export default function (context_path) { + let root = path.join(__dirname, "../../../.."); + let dir = path.join(root, context_path); + let files = collectFilesInDirectory(dir).sort(); + let hashes = files.map(file => { + return sha256(file + "|" + fs.readFileSync(file, "utf-8")); + }); + + // Generate a new prefix every month to ensure the image stays buildable. + let now = new Date(); + let prefix = `${now.getUTCFullYear()}-${now.getUTCMonth() + 1}:`; + return sha256(prefix + hashes.join(",")); +} diff --git a/security/nss/automation/taskcluster/graph/src/extend.js b/security/nss/automation/taskcluster/graph/src/extend.js new file mode 100644 index 000000000..a6a8fcbe2 --- /dev/null +++ b/security/nss/automation/taskcluster/graph/src/extend.js @@ -0,0 +1,572 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import merge from "./merge"; +import * as queue from "./queue"; + +const LINUX_IMAGE = {name: "linux", path: "automation/taskcluster/docker"}; + +const WINDOWS_CHECKOUT_CMD = + "bash -c \"hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss || " + + "(sleep 2; hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss) || " + + "(sleep 5; hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss)\""; + +/*****************************************************************************/ + +queue.filter(task => { + if (task.group == "Builds") { + // Remove extra builds on {A,UB}San and ARM. + if (task.collection == "asan" || task.collection == "arm-debug" || + task.collection == "gyp-asan") { + return false; + } + + // Remove extra builds w/o libpkix for non-linux64-debug. + if (task.symbol == "noLibpkix" && + (task.platform != "linux64" || task.collection != "debug")) { + return false; + } + } + + if (task.tests == "bogo") { + // No BoGo tests on Windows. + if (task.platform == "windows2012-64") { + return false; + } + + // No BoGo tests on ARM. + if (task.collection == "arm-debug") { + return false; + } + } + + // GYP builds with -Ddisable_libpkix=1 by default. + if ((task.collection == "gyp" || task.collection == "gyp-asan") && + task.tests == "chains") { + return false; + } + + return true; +}); + +queue.map(task => { + if (task.collection == "asan" || task.collection == "gyp-asan") { + // CRMF and FIPS tests still leak, unfortunately. + if (task.tests == "crmf" || task.tests == "fips") { + task.env.ASAN_OPTIONS = "detect_leaks=0"; + } + } + + if (task.collection == "arm-debug") { + // These tests take quite some time on our poor ARM devices. + if (task.tests == "chains" || (task.tests == "ssl" && task.cycle == "standard")) { + task.maxRunTime = 14400; + } + } + + // Windows is slow. + if (task.platform == "windows2012-64" && task.tests == "chains") { + task.maxRunTime = 7200; + } + + // Enable TLS 1.3 for every task. + task.env = task.env || {}; + task.env.NSS_ENABLE_TLS_1_3 = "1"; + + return task; +}); + +/*****************************************************************************/ + +export default async function main() { + await scheduleLinux("Linux 32 (opt)", { + env: {BUILD_OPT: "1"}, + platform: "linux32", + image: LINUX_IMAGE + }); + + await scheduleLinux("Linux 32 (debug)", { + platform: "linux32", + collection: "debug", + image: LINUX_IMAGE + }); + + await scheduleLinux("Linux 64 (opt)", { + env: {USE_64: "1", BUILD_OPT: "1"}, + platform: "linux64", + image: LINUX_IMAGE + }); + + await scheduleLinux("Linux 64 (debug)", { + env: {USE_64: "1"}, + platform: "linux64", + collection: "debug", + image: LINUX_IMAGE + }); + + await scheduleLinux("Linux 64 (debug, gyp)", { + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/build_gyp.sh" + ], + platform: "linux64", + collection: "gyp", + image: LINUX_IMAGE + }); + + await scheduleLinux("Linux 64 (debug, gyp, asan, ubsan)", { + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/build_gyp.sh -g -v --ubsan --asan" + ], + env: { + ASAN_OPTIONS: "detect_odr_violation=0", // bug 1316276 + UBSAN_OPTIONS: "print_stacktrace=1", + NSS_DISABLE_ARENA_FREE_LIST: "1", + NSS_DISABLE_UNLOAD: "1", + CC: "clang", + CCC: "clang++" + }, + platform: "linux64", + collection: "gyp-asan", + image: LINUX_IMAGE + }); + + await scheduleLinux("Linux 64 (ASan, debug)", { + env: { + UBSAN_OPTIONS: "print_stacktrace=1", + NSS_DISABLE_ARENA_FREE_LIST: "1", + NSS_DISABLE_UNLOAD: "1", + CC: "clang", + CCC: "clang++", + USE_UBSAN: "1", + USE_ASAN: "1", + USE_64: "1" + }, + platform: "linux64", + collection: "asan", + image: LINUX_IMAGE + }); + + await scheduleWindows("Windows 2012 64 (opt)", { + env: {BUILD_OPT: "1"} + }); + + await scheduleWindows("Windows 2012 64 (debug)", { + collection: "debug" + }); + + await scheduleFuzzing(); + + await scheduleTestBuilds(); + + await scheduleTools(); + + await scheduleLinux("Linux 32 (ARM, debug)", { + image: "franziskus/nss-arm-ci", + provisioner: "localprovisioner", + collection: "arm-debug", + workerType: "nss-rpi", + platform: "linux32", + maxRunTime: 7200, + tier: 3 + }); +} + +/*****************************************************************************/ + +async function scheduleLinux(name, base) { + // Build base definition. + let build_base = merge({ + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/build.sh" + ], + artifacts: { + public: { + expires: 24 * 7, + type: "directory", + path: "/home/worker/artifacts" + } + }, + kind: "build", + symbol: "B" + }, base); + + // The task that builds NSPR+NSS. + let task_build = queue.scheduleTask(merge(build_base, {name})); + + // The task that generates certificates. + let task_cert = queue.scheduleTask(merge(build_base, { + name: "Certificates", + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/gen_certs.sh" + ], + parent: task_build, + symbol: "Certs" + })); + + // Schedule tests. + scheduleTests(task_build, task_cert, merge(base, { + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/run_tests.sh" + ] + })); + + // Extra builds. + let extra_base = merge({group: "Builds"}, build_base); + queue.scheduleTask(merge(extra_base, { + name: `${name} w/ clang-3.9`, + env: { + CC: "clang", + CCC: "clang++", + }, + symbol: "clang-3.9" + })); + + queue.scheduleTask(merge(extra_base, { + name: `${name} w/ gcc-4.8`, + env: { + CC: "gcc-4.8", + CCC: "g++-4.8" + }, + symbol: "gcc-4.8" + })); + + queue.scheduleTask(merge(extra_base, { + name: `${name} w/ gcc-6.1`, + env: { + CC: "gcc-6", + CCC: "g++-6" + }, + symbol: "gcc-6.1" + })); + + queue.scheduleTask(merge(extra_base, { + name: `${name} w/ NSS_DISABLE_LIBPKIX=1`, + env: {NSS_DISABLE_LIBPKIX: "1"}, + symbol: "noLibpkix" + })); + + return queue.submit(); +} + +/*****************************************************************************/ + +async function scheduleFuzzing() { + let base = { + env: { + // bug 1316276 + ASAN_OPTIONS: "allocator_may_return_null=1:detect_odr_violation=0", + UBSAN_OPTIONS: "print_stacktrace=1", + NSS_DISABLE_ARENA_FREE_LIST: "1", + NSS_DISABLE_UNLOAD: "1", + CC: "clang", + CCC: "clang++" + }, + platform: "linux64", + collection: "fuzz", + image: LINUX_IMAGE + }; + + // Build base definition. + let build_base = merge({ + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && " + + "nss/automation/taskcluster/scripts/build_gyp.sh -g -v --fuzz" + ], + artifacts: { + public: { + expires: 24 * 7, + type: "directory", + path: "/home/worker/artifacts" + } + }, + kind: "build", + symbol: "B" + }, base); + + // The task that builds NSPR+NSS. + let task_build = queue.scheduleTask(merge(build_base, { + name: "Linux x64 (debug, fuzz)" + })); + + // Schedule tests. + queue.scheduleTask(merge(base, { + parent: task_build, + name: "Gtests", + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/run_tests.sh" + ], + env: {GTESTFILTER: "*Fuzz*"}, + tests: "ssl_gtests gtests", + cycle: "standard", + symbol: "Gtest", + kind: "test" + })); + + queue.scheduleTask(merge(base, { + parent: task_build, + name: "Cert", + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/fuzz.sh " + + "cert nss/fuzz/corpus/cert -max_total_time=300" + ], + // Need a privileged docker container to remove this. + env: {ASAN_OPTIONS: "detect_leaks=0"}, + symbol: "SCert", + kind: "test" + })); + + queue.scheduleTask(merge(base, { + parent: task_build, + name: "SPKI", + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/fuzz.sh " + + "spki nss/fuzz/corpus/spki -max_total_time=300" + ], + // Need a privileged docker container to remove this. + env: {ASAN_OPTIONS: "detect_leaks=0"}, + symbol: "SPKI", + kind: "test" + })); + + return queue.submit(); +} + +/*****************************************************************************/ + +async function scheduleTestBuilds() { + let base = { + platform: "linux64", + collection: "gyp", + group: "Test", + image: LINUX_IMAGE + }; + + // Build base definition. + let build = merge({ + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && " + + "nss/automation/taskcluster/scripts/build_gyp.sh -g -v --test" + ], + artifacts: { + public: { + expires: 24 * 7, + type: "directory", + path: "/home/worker/artifacts" + } + }, + kind: "build", + symbol: "B", + name: "Linux 64 (debug, gyp, test)" + }, base); + + // The task that builds NSPR+NSS. + let task_build = queue.scheduleTask(build); + + // Schedule tests. + queue.scheduleTask(merge(base, { + parent: task_build, + name: "mpi", + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/run_tests.sh" + ], + tests: "mpi", + cycle: "standard", + symbol: "mpi", + kind: "test" + })); + + return queue.submit(); +} + + +/*****************************************************************************/ + +async function scheduleWindows(name, base) { + base = merge(base, { + workerType: "nss-win2012r2", + platform: "windows2012-64", + env: { + PATH: "c:\\mozilla-build\\python;c:\\mozilla-build\\msys\\local\\bin;" + + "c:\\mozilla-build\\7zip;c:\\mozilla-build\\info-zip;" + + "c:\\mozilla-build\\python\\Scripts;c:\\mozilla-build\\yasm;" + + "c:\\mozilla-build\\msys\\bin;c:\\Windows\\system32;" + + "c:\\mozilla-build\\upx391w;c:\\mozilla-build\\moztools-x64\\bin;" + + "c:\\mozilla-build\\wget", + DOMSUF: "localdomain", + HOST: "localhost", + USE_64: "1" + } + }); + + // Build base definition. + let build_base = merge(base, { + command: [ + WINDOWS_CHECKOUT_CMD, + "bash -c nss/automation/taskcluster/windows/build.sh" + ], + artifacts: [{ + expires: 24 * 7, + type: "directory", + path: "public\\build" + }], + kind: "build", + symbol: "B" + }); + + // The task that builds NSPR+NSS. + let task_build = queue.scheduleTask(merge(build_base, {name})); + + // The task that generates certificates. + let task_cert = queue.scheduleTask(merge(build_base, { + name: "Certificates", + command: [ + WINDOWS_CHECKOUT_CMD, + "bash -c nss/automation/taskcluster/windows/gen_certs.sh" + ], + parent: task_build, + symbol: "Certs" + })); + + // Schedule tests. + scheduleTests(task_build, task_cert, merge(base, { + command: [ + WINDOWS_CHECKOUT_CMD, + "bash -c nss/automation/taskcluster/windows/run_tests.sh" + ] + })); + + return queue.submit(); +} + +/*****************************************************************************/ + +function scheduleTests(task_build, task_cert, test_base) { + test_base = merge({kind: "test"}, test_base); + + // Schedule tests that do NOT need certificates. + let no_cert_base = merge(test_base, {parent: task_build}); + queue.scheduleTask(merge(no_cert_base, { + name: "Gtests", symbol: "Gtest", tests: "ssl_gtests gtests", cycle: "standard" + })); + queue.scheduleTask(merge(no_cert_base, { + name: "Bogo tests", symbol: "Bogo", tests: "bogo", cycle: "standard" + })); + queue.scheduleTask(merge(no_cert_base, { + name: "Chains tests", symbol: "Chains", tests: "chains" + })); + queue.scheduleTask(merge(no_cert_base, { + name: "Cipher tests", symbol: "Cipher", tests: "cipher" + })); + queue.scheduleTask(merge(no_cert_base, { + name: "EC tests", symbol: "EC", tests: "ec" + })); + queue.scheduleTask(merge(no_cert_base, { + name: "Lowhash tests", symbol: "Lowhash", tests: "lowhash" + })); + queue.scheduleTask(merge(no_cert_base, { + name: "SDR tests", symbol: "SDR", tests: "sdr" + })); + + // Schedule tests that need certificates. + let cert_base = merge(test_base, {parent: task_cert}); + queue.scheduleTask(merge(cert_base, { + name: "CRMF tests", symbol: "CRMF", tests: "crmf" + })); + queue.scheduleTask(merge(cert_base, { + name: "DB tests", symbol: "DB", tests: "dbtests" + })); + queue.scheduleTask(merge(cert_base, { + name: "FIPS tests", symbol: "FIPS", tests: "fips" + })); + queue.scheduleTask(merge(cert_base, { + name: "Merge tests", symbol: "Merge", tests: "merge" + })); + queue.scheduleTask(merge(cert_base, { + name: "S/MIME tests", symbol: "SMIME", tests: "smime" + })); + queue.scheduleTask(merge(cert_base, { + name: "Tools tests", symbol: "Tools", tests: "tools" + })); + + // SSL tests, need certificates too. + let ssl_base = merge(cert_base, {tests: "ssl", group: "SSL"}); + queue.scheduleTask(merge(ssl_base, { + name: "SSL tests (standard)", symbol: "standard", cycle: "standard" + })); + queue.scheduleTask(merge(ssl_base, { + name: "SSL tests (pkix)", symbol: "pkix", cycle: "pkix" + })); + queue.scheduleTask(merge(ssl_base, { + name: "SSL tests (sharedb)", symbol: "sharedb", cycle: "sharedb" + })); + queue.scheduleTask(merge(ssl_base, { + name: "SSL tests (upgradedb)", symbol: "upgradedb", cycle: "upgradedb" + })); +} + +/*****************************************************************************/ + +async function scheduleTools() { + let base = { + image: LINUX_IMAGE, + platform: "nss-tools", + kind: "test" + }; + + queue.scheduleTask(merge(base, { + symbol: "clang-format-3.9", + name: "clang-format-3.9", + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/run_clang_format.sh" + ] + })); + + queue.scheduleTask(merge(base, { + symbol: "scan-build-3.9", + name: "scan-build-3.9", + env: { + USE_64: "1", + CC: "clang", + CCC: "clang++", + }, + artifacts: { + public: { + expires: 24 * 7, + type: "directory", + path: "/home/worker/artifacts" + } + }, + command: [ + "/bin/bash", + "-c", + "bin/checkout.sh && nss/automation/taskcluster/scripts/run_scan_build.sh" + ] + })); + + return queue.submit(); +} diff --git a/security/nss/automation/taskcluster/graph/src/image_builder.js b/security/nss/automation/taskcluster/graph/src/image_builder.js new file mode 100644 index 000000000..bc90e0242 --- /dev/null +++ b/security/nss/automation/taskcluster/graph/src/image_builder.js @@ -0,0 +1,62 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import * as queue from "./queue"; +import context_hash from "./context_hash"; +import taskcluster from "taskcluster-client"; + +async function taskHasImageArtifact(taskId) { + let queue = new taskcluster.Queue(); + let {artifacts} = await queue.listLatestArtifacts(taskId); + return artifacts.some(artifact => artifact.name == "public/image.tar"); +} + +async function findTaskWithImageArtifact(ns) { + let index = new taskcluster.Index(); + let {taskId} = await index.findTask(ns); + let has_image = await taskHasImageArtifact(taskId); + return has_image ? taskId : null; +} + +export async function findTask({name, path}) { + let hash = await context_hash(path); + let ns = `docker.images.v1.${process.env.TC_PROJECT}.${name}.hash.${hash}`; + return findTaskWithImageArtifact(ns).catch(() => null); +} + +export async function buildTask({name, path}) { + let hash = await context_hash(path); + let ns = `docker.images.v1.${process.env.TC_PROJECT}.${name}.hash.${hash}`; + + return { + name: "Image Builder", + image: "taskcluster/image_builder:0.1.5", + routes: ["index." + ns], + env: { + HEAD_REPOSITORY: process.env.NSS_HEAD_REPOSITORY, + BASE_REPOSITORY: process.env.NSS_HEAD_REPOSITORY, + HEAD_REV: process.env.NSS_HEAD_REVISION, + HEAD_REF: process.env.NSS_HEAD_REVISION, + PROJECT: process.env.TC_PROJECT, + CONTEXT_PATH: path, + HASH: hash + }, + artifacts: { + "public/image.tar": { + type: "file", + expires: 24 * 90, + path: "/artifacts/image.tar" + } + }, + command: [ + "/bin/bash", + "-c", + "/home/worker/bin/build_image.sh" + ], + platform: "nss-decision", + features: ["dind"], + kind: "build", + symbol: "I" + }; +} diff --git a/security/nss/automation/taskcluster/graph/src/index.js b/security/nss/automation/taskcluster/graph/src/index.js new file mode 100644 index 000000000..4153e1b18 --- /dev/null +++ b/security/nss/automation/taskcluster/graph/src/index.js @@ -0,0 +1,14 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import * as try_syntax from "./try_syntax"; +import extend from "./extend"; + +// Init try syntax filter. +if (process.env.TC_PROJECT == "nss-try") { + try_syntax.initFilter(); +} + +// Extend the task graph. +extend().catch(console.error); diff --git a/security/nss/automation/taskcluster/graph/src/merge.js b/security/nss/automation/taskcluster/graph/src/merge.js new file mode 100644 index 000000000..17043dd8e --- /dev/null +++ b/security/nss/automation/taskcluster/graph/src/merge.js @@ -0,0 +1,10 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import {recursive as merge} from "merge"; + +// We always want to clone. +export default function (...args) { + return merge(true, ...args); +} diff --git a/security/nss/automation/taskcluster/graph/src/queue.js b/security/nss/automation/taskcluster/graph/src/queue.js new file mode 100644 index 000000000..2a4a7b3fe --- /dev/null +++ b/security/nss/automation/taskcluster/graph/src/queue.js @@ -0,0 +1,242 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import {clone} from "merge"; +import merge from "./merge"; +import slugid from "slugid"; +import taskcluster from "taskcluster-client"; +import * as image_builder from "./image_builder"; + +let maps = []; +let filters = []; + +let tasks = new Map(); +let image_tasks = new Map(); + +let queue = new taskcluster.Queue({ + baseUrl: "http://taskcluster/queue/v1" +}); + +function fromNow(hours) { + let d = new Date(); + d.setHours(d.getHours() + (hours|0)); + return d.toJSON(); +} + +function parseRoutes(routes) { + return [ + `tc-treeherder.v2.${process.env.TC_PROJECT}.${process.env.NSS_HEAD_REVISION}.${process.env.NSS_PUSHLOG_ID}`, + ...routes + ]; +} + +function parseFeatures(list) { + return list.reduce((map, feature) => { + map[feature] = true; + return map; + }, {}); +} + +function parseArtifacts(artifacts) { + let copy = clone(artifacts); + Object.keys(copy).forEach(key => { + copy[key].expires = fromNow(copy[key].expires); + }); + return copy; +} + +function parseCollection(name) { + let collection = {}; + collection[name] = true; + return collection; +} + +function parseTreeherder(def) { + let treeherder = { + build: { + platform: def.platform + }, + machine: { + platform: def.platform + }, + symbol: def.symbol, + jobKind: def.kind + }; + + if (def.group) { + treeherder.groupSymbol = def.group; + } + + if (def.collection) { + treeherder.collection = parseCollection(def.collection); + } + + if (def.tier) { + treeherder.tier = def.tier; + } + + return treeherder; +} + +function convertTask(def) { + let dependencies = []; + + let env = merge({ + NSS_HEAD_REPOSITORY: process.env.NSS_HEAD_REPOSITORY, + NSS_HEAD_REVISION: process.env.NSS_HEAD_REVISION + }, def.env || {}); + + if (def.parent) { + dependencies.push(def.parent); + env.TC_PARENT_TASK_ID = def.parent; + } + + if (def.tests) { + env.NSS_TESTS = def.tests; + } + + if (def.cycle) { + env.NSS_CYCLES = def.cycle; + } + + let payload = { + env, + command: def.command, + maxRunTime: def.maxRunTime || 3600 + }; + + if (def.image) { + payload.image = def.image; + } + + if (def.features) { + payload.features = parseFeatures(def.features); + } + + if (def.artifacts) { + payload.artifacts = parseArtifacts(def.artifacts); + } + + return { + provisionerId: def.provisioner || "aws-provisioner-v1", + workerType: def.workerType || "hg-worker", + schedulerId: "task-graph-scheduler", + + created: fromNow(0), + deadline: fromNow(24), + + dependencies, + routes: parseRoutes(def.routes || []), + + metadata: { + name: def.name, + description: def.name, + owner: process.env.TC_OWNER, + source: process.env.TC_SOURCE + }, + + payload, + + extra: { + treeherder: parseTreeherder(def) + } + }; +} + +export function map(fun) { + maps.push(fun); +} + +export function filter(fun) { + filters.push(fun); +} + +export function scheduleTask(def) { + let taskId = slugid.v4(); + tasks.set(taskId, merge({}, def)); + return taskId; +} + +export async function submit() { + let promises = new Map(); + + for (let [taskId, task] of tasks) { + // Allow filtering tasks before we schedule them. + if (!filters.every(filter => filter(task))) { + continue; + } + + // Allow changing tasks before we schedule them. + maps.forEach(map => { task = map(merge({}, task)) }); + + let log_id = `${task.name} @ ${task.platform}[${task.collection || "opt"}]`; + console.log(`+ Submitting ${log_id}.`); + + let parent = task.parent; + + // Convert the task definition. + task = await convertTask(task); + + // Convert the docker image definition. + let image_def = task.payload.image; + if (image_def && image_def.hasOwnProperty("path")) { + let key = `${image_def.name}:${image_def.path}`; + let data = {}; + + // Check the cache first. + if (image_tasks.has(key)) { + data = image_tasks.get(key); + } else { + data.taskId = await image_builder.findTask(image_def); + data.isPending = !data.taskId; + + // No task found. + if (data.isPending) { + let image_task = await image_builder.buildTask(image_def); + + // Schedule a new image builder task immediately. + data.taskId = slugid.v4(); + + try { + await queue.createTask(data.taskId, convertTask(image_task)); + } catch (e) { + console.error("! FAIL: Scheduling image builder task failed."); + continue; /* Skip this task on failure. */ + } + } + + // Store in cache. + image_tasks.set(key, data); + } + + if (data.isPending) { + task.dependencies.push(data.taskId); + } + + task.payload.image = { + path: "public/image.tar", + taskId: data.taskId, + type: "task-image" + }; + } + + // Wait for the parent task to be created before scheduling dependants. + let predecessor = parent ? promises.get(parent) : Promise.resolve(); + + promises.set(taskId, predecessor.then(() => { + // Schedule the task. + return queue.createTask(taskId, task).catch(err => { + console.error(`! FAIL: Scheduling ${log_id} failed.`, err); + }); + })); + } + + // Wait for all requests to finish. + if (promises.length) { + await Promise.all([...promises.values()]); + console.log("=== Total:", promises.length, "tasks. ==="); + } + + tasks.clear(); +} diff --git a/security/nss/automation/taskcluster/graph/src/try_syntax.js b/security/nss/automation/taskcluster/graph/src/try_syntax.js new file mode 100644 index 000000000..695c9e92f --- /dev/null +++ b/security/nss/automation/taskcluster/graph/src/try_syntax.js @@ -0,0 +1,159 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +import * as queue from "./queue"; +import intersect from "intersect"; +import parse_args from "minimist"; + +function parseOptions(opts) { + opts = parse_args(opts.split(/\s+/), { + default: {build: "do", platform: "all", unittests: "none", tools: "none"}, + alias: {b: "build", p: "platform", u: "unittests", t: "tools", e: "extra-builds"}, + string: ["build", "platform", "unittests", "tools", "extra-builds"] + }); + + // Parse build types (d=debug, o=opt). + let builds = intersect(opts.build.split(""), ["d", "o"]); + + // If the given value is nonsense default to debug and opt builds. + if (builds.length == 0) { + builds = ["d", "o"]; + } + + // Parse platforms. + let allPlatforms = ["linux", "linux64", "linux64-asan", "win64", "arm", + "linux64-gyp", "linux64-gyp-asan", "linux64-fuzz"]; + let platforms = intersect(opts.platform.split(/\s*,\s*/), allPlatforms); + + // If the given value is nonsense or "none" default to all platforms. + if (platforms.length == 0 && opts.platform != "none") { + platforms = allPlatforms; + } + + // Parse unit tests. + let aliases = {"gtests": "gtest"}; + let allUnitTests = ["bogo", "crmf", "chains", "cipher", "db", "ec", "fips", + "gtest", "lowhash", "merge", "sdr", "smime", "tools", + "ssl", "mpi", "scert", "spki"]; + let unittests = intersect(opts.unittests.split(/\s*,\s*/).map(t => { + return aliases[t] || t; + }), allUnitTests); + + // If the given value is "all" run all tests. + // If it's nonsense then don't run any tests. + if (opts.unittests == "all") { + unittests = allUnitTests; + } else if (unittests.length == 0) { + unittests = []; + } + + // Parse tools. + let allTools = ["clang-format", "scan-build"]; + let tools = intersect(opts.tools.split(/\s*,\s*/), allTools); + + // If the given value is "all" run all tools. + // If it's nonsense then don't run any tools. + if (opts.tools == "all") { + tools = allTools; + } else if (tools.length == 0) { + tools = []; + } + + return { + builds: builds, + platforms: platforms, + unittests: unittests, + extra: (opts.e == "all"), + tools: tools + }; +} + +function filter(opts) { + return function (task) { + // Filter tools. We can immediately return here as those + // are not affected by platform or build type selectors. + if (task.platform == "nss-tools") { + return opts.tools.some(tool => { + return task.symbol.toLowerCase().startsWith(tool); + }); + } + + // Filter unit tests. + if (task.tests) { + let found = opts.unittests.some(test => { + // TODO: think of something more intelligent here. + if (task.symbol.toLowerCase().startsWith("mpi") && test == "mpi") { + return true; + } + return (task.group || task.symbol).toLowerCase().startsWith(test); + }); + + if (!found) { + return false; + } + } + + // Filter extra builds. + if (task.group == "Builds" && !opts.extra) { + return false; + } + + let coll = name => name == (task.collection || "opt"); + + // Filter by platform. + let found = opts.platforms.some(platform => { + let aliases = { + "linux": "linux32", + "linux64-asan": "linux64", + "linux64-fuzz": "linux64", + "linux64-gyp": "linux64", + "linux64-gyp-asan": "linux64", + "win64": "windows2012-64", + "arm": "linux32" + }; + + // Check the platform name. + let keep = (task.platform == (aliases[platform] || platform)); + + // Additional checks. + if (platform == "linux64-asan") { + keep &= coll("asan"); + } else if (platform == "arm") { + keep &= coll("arm-opt") || coll("arm-debug"); + } else if (platform == "linux64-gyp") { + keep &= coll("gyp"); + } else if (platform == "linux64-gyp-asan") { + keep &= coll("gyp-asan"); + } else if (platform == "linux64-fuzz") { + keep &= coll("fuzz"); + } else { + keep &= coll("opt") || coll("debug"); + } + + return keep; + }); + + if (!found) { + return false; + } + + // Finally, filter by build type. + let isDebug = coll("debug") || coll("asan") || coll("arm-debug") || + coll("gyp") || coll("fuzz"); + return (isDebug && opts.builds.includes("d")) || + (!isDebug && opts.builds.includes("o")); + } +} + +export function initFilter() { + let comment = process.env.TC_COMMENT || ""; + + // Check for try syntax in changeset comment. + let match = comment.match(/^\s*try:\s*(.*)\s*$/); + + // Add try syntax filter. + if (match) { + queue.filter(filter(parseOptions(match[1]))); + } +} |