summaryrefslogtreecommitdiffstats
path: root/security/nss/automation
diff options
context:
space:
mode:
authorwolfbeast <mcwerewolf@gmail.com>2018-02-24 00:43:43 +0100
committerwolfbeast <mcwerewolf@gmail.com>2018-02-24 00:43:43 +0100
commitdc7ceccf8bc0950e4daa5f0e7736d4c204526fc0 (patch)
treec70154c611b0f03bdcba029f1c51c48e5f69eb36 /security/nss/automation
parent6f93b00559c8fe5b8dfaf59fcbaac8f1e0dd3366 (diff)
parent788c588bf6f8907dc57b01cd1c86239215d3c661 (diff)
downloadUXP-dc7ceccf8bc0950e4daa5f0e7736d4c204526fc0.tar
UXP-dc7ceccf8bc0950e4daa5f0e7736d4c204526fc0.tar.gz
UXP-dc7ceccf8bc0950e4daa5f0e7736d4c204526fc0.tar.lz
UXP-dc7ceccf8bc0950e4daa5f0e7736d4c204526fc0.tar.xz
UXP-dc7ceccf8bc0950e4daa5f0e7736d4c204526fc0.zip
Merge branch 'TLS-1.3'
Diffstat (limited to 'security/nss/automation')
-rw-r--r--security/nss/automation/abi-check/expected-report-libnspr4.so.txt8
-rw-r--r--security/nss/automation/abi-check/expected-report-libssl3.so.txt3
-rw-r--r--security/nss/automation/abi-check/previous-nss-release2
-rwxr-xr-xsecurity/nss/automation/buildbot-slave/build.sh37
-rw-r--r--security/nss/automation/clang-format/run_clang_format.sh21
-rw-r--r--security/nss/automation/clang-format/setup.sh4
-rw-r--r--security/nss/automation/release/nspr-version.txt2
-rw-r--r--security/nss/automation/taskcluster/docker-clang-3.9/setup.sh4
-rw-r--r--security/nss/automation/taskcluster/docker-decision/Dockerfile3
-rw-r--r--security/nss/automation/taskcluster/docker-decision/bin/checkout.sh5
-rw-r--r--security/nss/automation/taskcluster/docker-gcc-4.4/Dockerfile30
-rw-r--r--security/nss/automation/taskcluster/docker-gcc-4.4/bin/checkout.sh20
-rw-r--r--security/nss/automation/taskcluster/docker-gcc-4.4/setup.sh30
-rw-r--r--security/nss/automation/taskcluster/docker-hacl/Dockerfile30
-rw-r--r--security/nss/automation/taskcluster/docker-hacl/bin/checkout.sh20
-rw-r--r--security/nss/automation/taskcluster/docker-hacl/license.txt15
-rw-r--r--security/nss/automation/taskcluster/docker-hacl/setup-user.sh26
-rw-r--r--security/nss/automation/taskcluster/docker-hacl/setup.sh30
-rw-r--r--security/nss/automation/taskcluster/docker/setup.sh4
-rw-r--r--security/nss/automation/taskcluster/graph/src/context_hash.js16
-rw-r--r--security/nss/automation/taskcluster/graph/src/extend.js232
-rw-r--r--security/nss/automation/taskcluster/graph/src/image_builder.js11
-rw-r--r--security/nss/automation/taskcluster/graph/src/try_syntax.js9
-rw-r--r--security/nss/automation/taskcluster/image_builder/Dockerfile23
-rw-r--r--security/nss/automation/taskcluster/image_builder/VERSION1
-rw-r--r--security/nss/automation/taskcluster/image_builder/bin/checkout.sh15
-rwxr-xr-xsecurity/nss/automation/taskcluster/scripts/build_gyp.sh9
-rw-r--r--security/nss/automation/taskcluster/scripts/build_image.sh24
-rwxr-xr-xsecurity/nss/automation/taskcluster/scripts/gen_certs.sh9
-rw-r--r--security/nss/automation/taskcluster/scripts/run_hacl.sh40
-rw-r--r--security/nss/automation/taskcluster/scripts/split.sh6
-rw-r--r--security/nss/automation/taskcluster/windows/releng.manifest8
-rw-r--r--security/nss/automation/taskcluster/windows/setup.sh6
-rw-r--r--security/nss/automation/taskcluster/windows/setup32.sh6
-rw-r--r--security/nss/automation/taskcluster/windows/setup64.sh6
35 files changed, 642 insertions, 73 deletions
diff --git a/security/nss/automation/abi-check/expected-report-libnspr4.so.txt b/security/nss/automation/abi-check/expected-report-libnspr4.so.txt
index e69de29bb..44f52325f 100644
--- a/security/nss/automation/abi-check/expected-report-libnspr4.so.txt
+++ b/security/nss/automation/abi-check/expected-report-libnspr4.so.txt
@@ -0,0 +1,8 @@
+Functions changes summary: 1 Removed, 0 Changed, 0 Added function
+Variables changes summary: 0 Removed, 0 Changed, 0 Added variable
+
+1 Removed function:
+
+ 'function void PR_EXPERIMENTAL_ONLY_IN_4_17_GetOverlappedIOHandle(void**)' {PR_EXPERIMENTAL_ONLY_IN_4_17_GetOverlappedIOHandle}
+
+
diff --git a/security/nss/automation/abi-check/expected-report-libssl3.so.txt b/security/nss/automation/abi-check/expected-report-libssl3.so.txt
index e69de29bb..2a093094f 100644
--- a/security/nss/automation/abi-check/expected-report-libssl3.so.txt
+++ b/security/nss/automation/abi-check/expected-report-libssl3.so.txt
@@ -0,0 +1,3 @@
+Functions changes summary: 0 Removed, 0 Changed (5 filtered out), 0 Added function
+Variables changes summary: 0 Removed, 0 Changed, 0 Added variable
+
diff --git a/security/nss/automation/abi-check/previous-nss-release b/security/nss/automation/abi-check/previous-nss-release
index b8d28cde0..a91a569f5 100644
--- a/security/nss/automation/abi-check/previous-nss-release
+++ b/security/nss/automation/abi-check/previous-nss-release
@@ -1 +1 @@
-NSS_3_31_BRANCH
+NSS_3_34_BRANCH
diff --git a/security/nss/automation/buildbot-slave/build.sh b/security/nss/automation/buildbot-slave/build.sh
index 3fc914803..844254dae 100755
--- a/security/nss/automation/buildbot-slave/build.sh
+++ b/security/nss/automation/buildbot-slave/build.sh
@@ -236,11 +236,14 @@ check_abi()
BASE_NSPR=NSPR_$(head -1 ${HGDIR}/baseline/nss/automation/release/nspr-version.txt | cut -d . -f 1-2 | tr . _)_BRANCH
hg clone -u "${BASE_NSPR}" "${HGDIR}/nspr" "${HGDIR}/baseline/nspr"
if [ $? -ne 0 ]; then
- echo "invalid tag ${BASE_NSPR} derived from ${BASE_NSS} automation/release/nspr-version.txt"
- return 1
+ echo "nonexisting tag ${BASE_NSPR} derived from ${BASE_NSS} automation/release/nspr-version.txt"
+ # Assume that version hasn't been released yet, fall back to trunk
+ pushd "${HGDIR}/baseline/nspr"
+ hg update default
+ popd
fi
- print_log "######## building older NSPR/NSS ########"
+ print_log "######## building baseline NSPR/NSS ########"
pushd ${HGDIR}/baseline/nss
print_log "$ ${MAKE} ${NSS_BUILD_TARGET}"
@@ -253,26 +256,44 @@ check_abi()
fi
popd
+ ABI_PROBLEM_FOUND=0
ABI_REPORT=${OUTPUTDIR}/abi-diff.txt
rm -f ${ABI_REPORT}
PREVDIST=${HGDIR}/baseline/dist
NEWDIST=${HGDIR}/dist
ALL_SOs="libfreebl3.so libfreeblpriv3.so libnspr4.so libnss3.so libnssckbi.so libnssdbm3.so libnsssysinit.so libnssutil3.so libplc4.so libplds4.so libsmime3.so libsoftokn3.so libssl3.so"
for SO in ${ALL_SOs}; do
- if [ ! -f nss/automation/abi-check/expected-report-$SO.txt ]; then
- touch nss/automation/abi-check/expected-report-$SO.txt
+ if [ ! -f ${HGDIR}/nss/automation/abi-check/expected-report-$SO.txt ]; then
+ touch ${HGDIR}/nss/automation/abi-check/expected-report-$SO.txt
fi
abidiff --hd1 $PREVDIST/public/ --hd2 $NEWDIST/public \
$PREVDIST/*/lib/$SO $NEWDIST/*/lib/$SO \
- > nss/automation/abi-check/new-report-$SO.txt
- diff -u nss/automation/abi-check/expected-report-$SO.txt \
- nss/automation/abi-check/new-report-$SO.txt >> ${ABI_REPORT}
+ > ${HGDIR}/nss/automation/abi-check/new-report-$SO.txt
+ if [ $? -ne 0 ]; then
+ ABI_PROBLEM_FOUND=1
+ print_log "FAILED to run abidiff {$PREVDIST , $NEWDIST} for $SO, or failed writing to ${HGDIR}/nss/automation/abi-check/new-report-$SO.txt"
+ fi
+ if [ ! -f ${HGDIR}/nss/automation/abi-check/expected-report-$SO.txt ]; then
+ ABI_PROBLEM_FOUND=1
+ print_log "FAILED to access report file: ${HGDIR}/nss/automation/abi-check/expected-report-$SO.txt"
+ fi
+
+ diff -wB -u ${HGDIR}/nss/automation/abi-check/expected-report-$SO.txt \
+ ${HGDIR}/nss/automation/abi-check/new-report-$SO.txt >> ${ABI_REPORT}
+ if [ ! -f ${ABI_REPORT} ]; then
+ ABI_PROBLEM_FOUND=1
+ print_log "FAILED to compare exepcted and new report: ${HGDIR}/nss/automation/abi-check/new-report-$SO.txt"
+ fi
done
if [ -s ${ABI_REPORT} ]; then
print_log "FAILED: there are new unexpected ABI changes"
cat ${ABI_REPORT}
return 1
+ elif [ $ABI_PROBLEM_FOUND -ne 0 ]; then
+ print_log "FAILED: failure executing the ABI checks"
+ cat ${ABI_REPORT}
+ return 1
fi
return 0
diff --git a/security/nss/automation/clang-format/run_clang_format.sh b/security/nss/automation/clang-format/run_clang_format.sh
index 2ba5ebeb1..378b00ff0 100644
--- a/security/nss/automation/clang-format/run_clang_format.sh
+++ b/security/nss/automation/clang-format/run_clang_format.sh
@@ -6,6 +6,8 @@ if [[ $(id -u) -eq 0 ]]; then
exec su worker -c "$0 $*"
fi
+set -e
+
# Apply clang-format on the provided folder and verify that this doesn't change any file.
# If any file differs after formatting, the script eventually exits with 1.
# Any differences between formatted and unformatted files is printed to stdout to give a hint what's wrong.
@@ -21,17 +23,16 @@ blacklist=(
"./lib/zlib" \
"./lib/sqlite" \
"./gtests/google_test" \
- "./.hg" \
"./out" \
)
-top="$(dirname $0)/../.."
-cd "$top"
+top=$(cd "$(dirname $0)/../.."; pwd -P)
if [ $# -gt 0 ]; then
dirs=("$@")
else
- dirs=($(find . -maxdepth 2 -mindepth 1 -type d ! -path . \( ! -regex '.*/' \)))
+ cd "$top"
+ dirs=($(find . -maxdepth 2 -mindepth 1 -type d ! -path '*/.*' -print))
fi
format_folder()
@@ -46,20 +47,20 @@ format_folder()
}
for dir in "${dirs[@]}"; do
- if format_folder "$dir" ; then
+ if format_folder "$dir"; then
c="${dir//[^\/]}"
echo "formatting $dir ..."
- depth=""
+ depth=()
if [ "${#c}" == "1" ]; then
- depth="-maxdepth 1"
+ depth+=(-maxdepth 1)
fi
- find "$dir" $depth -type f \( -name '*.[ch]' -o -name '*.cc' \) -exec clang-format -i {} \+
+ find "$dir" "${depth[@]}" -type f \( -name '*.[ch]' -o -name '*.cc' \) -exec clang-format -i {} \+
fi
done
TMPFILE=$(mktemp /tmp/$(basename $0).XXXXXX)
-trap 'rm $TMPFILE' exit
-if (cd $(dirname $0); hg root >/dev/null 2>&1); then
+trap 'rm -f $TMPFILE' exit
+if [[ -d "$top/.hg" ]]; then
hg diff --git "$top" | tee $TMPFILE
else
git -C "$top" diff | tee $TMPFILE
diff --git a/security/nss/automation/clang-format/setup.sh b/security/nss/automation/clang-format/setup.sh
index 9b2480e90..beac9e905 100644
--- a/security/nss/automation/clang-format/setup.sh
+++ b/security/nss/automation/clang-format/setup.sh
@@ -17,8 +17,8 @@ apt_packages+=('locales')
apt-get install -y --no-install-recommends ${apt_packages[@]}
# Download clang.
-curl -L http://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz -o clang.tar.xz
-curl -L http://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz.sig -o clang.tar.xz.sig
+curl -L https://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz -o clang.tar.xz
+curl -L https://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz.sig -o clang.tar.xz.sig
# Verify the signature.
gpg --keyserver pool.sks-keyservers.net --recv-keys B6C8F98282B944E3B0D5C2530FC3042E345AD05D
gpg --verify clang.tar.xz.sig
diff --git a/security/nss/automation/release/nspr-version.txt b/security/nss/automation/release/nspr-version.txt
index 98783a615..01eeb3615 100644
--- a/security/nss/automation/release/nspr-version.txt
+++ b/security/nss/automation/release/nspr-version.txt
@@ -1,4 +1,4 @@
-4.16
+4.18
# The first line of this file must contain the human readable NSPR
# version number, which is the minimum required version of NSPR
diff --git a/security/nss/automation/taskcluster/docker-clang-3.9/setup.sh b/security/nss/automation/taskcluster/docker-clang-3.9/setup.sh
index 7b7d534e6..3076667a6 100644
--- a/security/nss/automation/taskcluster/docker-clang-3.9/setup.sh
+++ b/security/nss/automation/taskcluster/docker-clang-3.9/setup.sh
@@ -25,8 +25,8 @@ apt-get -y update
apt-get install -y --no-install-recommends ${apt_packages[@]}
# Download clang.
-curl -LO http://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz
-curl -LO http://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz.sig
+curl -LO https://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz
+curl -LO https://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz.sig
# Verify the signature.
gpg --keyserver pool.sks-keyservers.net --recv-keys B6C8F98282B944E3B0D5C2530FC3042E345AD05D
gpg --verify *.tar.xz.sig
diff --git a/security/nss/automation/taskcluster/docker-decision/Dockerfile b/security/nss/automation/taskcluster/docker-decision/Dockerfile
index 35777c0b7..473ce64ba 100644
--- a/security/nss/automation/taskcluster/docker-decision/Dockerfile
+++ b/security/nss/automation/taskcluster/docker-decision/Dockerfile
@@ -12,6 +12,9 @@ RUN chmod +x /home/worker/bin/*
ADD setup.sh /tmp/setup.sh
RUN bash /tmp/setup.sh
+# Change user.
+USER worker
+
# Env variables.
ENV HOME /home/worker
ENV SHELL /bin/bash
diff --git a/security/nss/automation/taskcluster/docker-decision/bin/checkout.sh b/security/nss/automation/taskcluster/docker-decision/bin/checkout.sh
index 9167f6bda..0cdd2ac40 100644
--- a/security/nss/automation/taskcluster/docker-decision/bin/checkout.sh
+++ b/security/nss/automation/taskcluster/docker-decision/bin/checkout.sh
@@ -2,11 +2,6 @@
set -v -e -x
-if [ $(id -u) = 0 ]; then
- # Drop privileges by re-running this script.
- exec su worker $0
-fi
-
# Default values for testing.
REVISION=${NSS_HEAD_REVISION:-default}
REPOSITORY=${NSS_HEAD_REPOSITORY:-https://hg.mozilla.org/projects/nss}
diff --git a/security/nss/automation/taskcluster/docker-gcc-4.4/Dockerfile b/security/nss/automation/taskcluster/docker-gcc-4.4/Dockerfile
new file mode 100644
index 000000000..3330c007f
--- /dev/null
+++ b/security/nss/automation/taskcluster/docker-gcc-4.4/Dockerfile
@@ -0,0 +1,30 @@
+FROM ubuntu:14.04
+MAINTAINER Tim Taubert <ttaubert@mozilla.com>
+
+RUN useradd -d /home/worker -s /bin/bash -m worker
+WORKDIR /home/worker
+
+# Add build and test scripts.
+ADD bin /home/worker/bin
+RUN chmod +x /home/worker/bin/*
+
+# Install dependencies.
+ADD setup.sh /tmp/setup.sh
+RUN bash /tmp/setup.sh
+
+# Change user.
+USER worker
+
+# Env variables.
+ENV HOME /home/worker
+ENV SHELL /bin/bash
+ENV USER worker
+ENV LOGNAME worker
+ENV HOSTNAME taskcluster-worker
+ENV LANG en_US.UTF-8
+ENV LC_ALL en_US.UTF-8
+ENV HOST localhost
+ENV DOMSUF localdomain
+
+# Set a default command for debugging.
+CMD ["/bin/bash", "--login"]
diff --git a/security/nss/automation/taskcluster/docker-gcc-4.4/bin/checkout.sh b/security/nss/automation/taskcluster/docker-gcc-4.4/bin/checkout.sh
new file mode 100644
index 000000000..9167f6bda
--- /dev/null
+++ b/security/nss/automation/taskcluster/docker-gcc-4.4/bin/checkout.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -v -e -x
+
+if [ $(id -u) = 0 ]; then
+ # Drop privileges by re-running this script.
+ exec su worker $0
+fi
+
+# Default values for testing.
+REVISION=${NSS_HEAD_REVISION:-default}
+REPOSITORY=${NSS_HEAD_REPOSITORY:-https://hg.mozilla.org/projects/nss}
+
+# Clone NSS.
+for i in 0 2 5; do
+ sleep $i
+ hg clone -r $REVISION $REPOSITORY nss && exit 0
+ rm -rf nss
+done
+exit 1
diff --git a/security/nss/automation/taskcluster/docker-gcc-4.4/setup.sh b/security/nss/automation/taskcluster/docker-gcc-4.4/setup.sh
new file mode 100644
index 000000000..f6325d966
--- /dev/null
+++ b/security/nss/automation/taskcluster/docker-gcc-4.4/setup.sh
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+set -v -e -x
+
+# Update packages.
+export DEBIAN_FRONTEND=noninteractive
+apt-get -y update && apt-get -y upgrade
+
+apt_packages=()
+apt_packages+=('ca-certificates')
+apt_packages+=('g++-4.4')
+apt_packages+=('gcc-4.4')
+apt_packages+=('locales')
+apt_packages+=('make')
+apt_packages+=('mercurial')
+apt_packages+=('zlib1g-dev')
+
+# Install packages.
+apt-get -y update
+apt-get install -y --no-install-recommends ${apt_packages[@]}
+
+locale-gen en_US.UTF-8
+dpkg-reconfigure locales
+
+# Cleanup.
+rm -rf ~/.ccache ~/.cache
+apt-get autoremove -y
+apt-get clean
+apt-get autoclean
+rm $0
diff --git a/security/nss/automation/taskcluster/docker-hacl/Dockerfile b/security/nss/automation/taskcluster/docker-hacl/Dockerfile
new file mode 100644
index 000000000..e8a88f06c
--- /dev/null
+++ b/security/nss/automation/taskcluster/docker-hacl/Dockerfile
@@ -0,0 +1,30 @@
+FROM ubuntu:xenial
+
+MAINTAINER Franziskus Kiefer <franziskuskiefer@gmail.com>
+# Based on the HACL* image from Benjamin Beurdouche and
+# the original F* formula with Daniel Fabian
+
+# Pinned versions of HACL* (F* and KreMLin are pinned as submodules)
+ENV haclrepo https://github.com/mitls/hacl-star.git
+
+# Define versions of dependencies
+ENV opamv 4.04.2
+ENV haclversion dcd48329d535727dbde93877b124c5ec4a7a2b20
+
+# Install required packages and set versions
+ADD setup.sh /tmp/setup.sh
+RUN bash /tmp/setup.sh
+
+# Create user, add scripts.
+RUN useradd -ms /bin/bash worker
+WORKDIR /home/worker
+ADD bin /home/worker/bin
+RUN chmod +x /home/worker/bin/*
+USER worker
+
+# Build F*, HACL*, verify. Install a few more dependencies.
+ENV OPAMYES true
+ENV PATH "/home/worker/hacl-star/dependencies/z3/bin:$PATH"
+ADD setup-user.sh /tmp/setup-user.sh
+ADD license.txt /tmp/license.txt
+RUN bash /tmp/setup-user.sh
diff --git a/security/nss/automation/taskcluster/docker-hacl/bin/checkout.sh b/security/nss/automation/taskcluster/docker-hacl/bin/checkout.sh
new file mode 100644
index 000000000..9167f6bda
--- /dev/null
+++ b/security/nss/automation/taskcluster/docker-hacl/bin/checkout.sh
@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -v -e -x
+
+if [ $(id -u) = 0 ]; then
+ # Drop privileges by re-running this script.
+ exec su worker $0
+fi
+
+# Default values for testing.
+REVISION=${NSS_HEAD_REVISION:-default}
+REPOSITORY=${NSS_HEAD_REPOSITORY:-https://hg.mozilla.org/projects/nss}
+
+# Clone NSS.
+for i in 0 2 5; do
+ sleep $i
+ hg clone -r $REVISION $REPOSITORY nss && exit 0
+ rm -rf nss
+done
+exit 1
diff --git a/security/nss/automation/taskcluster/docker-hacl/license.txt b/security/nss/automation/taskcluster/docker-hacl/license.txt
new file mode 100644
index 000000000..03d25c4d3
--- /dev/null
+++ b/security/nss/automation/taskcluster/docker-hacl/license.txt
@@ -0,0 +1,15 @@
+/* Copyright 2016-2017 INRIA and Microsoft Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
diff --git a/security/nss/automation/taskcluster/docker-hacl/setup-user.sh b/security/nss/automation/taskcluster/docker-hacl/setup-user.sh
new file mode 100644
index 000000000..b8accaf58
--- /dev/null
+++ b/security/nss/automation/taskcluster/docker-hacl/setup-user.sh
@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+
+set -v -e -x
+
+# Prepare build (OCaml packages)
+opam init
+echo ". /home/worker/.opam/opam-init/init.sh > /dev/null 2> /dev/null || true" >> .bashrc
+opam switch -v ${opamv}
+opam install ocamlfind batteries sqlite3 fileutils yojson ppx_deriving_yojson zarith pprint menhir ulex process fix wasm stdint
+
+# Get the HACL* code
+git clone ${haclrepo} hacl-star
+git -C hacl-star checkout ${haclversion}
+
+# Prepare submodules, and build, verify, test, and extract c code
+# This caches the extracted c code (pins the HACL* version). All we need to do
+# on CI now is comparing the code in this docker image with the one in NSS.
+opam config exec -- make -C hacl-star prepare -j$(nproc)
+make -C hacl-star verify-nss -j$(nproc)
+make -C hacl-star -f Makefile.build snapshots/nss -j$(nproc)
+KOPTS="-funroll-loops 5" make -C hacl-star/code/curve25519 test -j$(nproc)
+make -C hacl-star/code/salsa-family test -j$(nproc)
+make -C hacl-star/code/poly1305 test -j$(nproc)
+
+# Cleanup.
+rm -rf ~/.ccache ~/.cache
diff --git a/security/nss/automation/taskcluster/docker-hacl/setup.sh b/security/nss/automation/taskcluster/docker-hacl/setup.sh
new file mode 100644
index 000000000..f5f8bd7d5
--- /dev/null
+++ b/security/nss/automation/taskcluster/docker-hacl/setup.sh
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+set -v -e -x
+
+# Update packages.
+export DEBIAN_FRONTEND=noninteractive
+apt-get -qq update
+apt-get install --yes libssl-dev libsqlite3-dev g++-5 gcc-5 m4 make opam pkg-config python libgmp3-dev cmake curl libtool-bin autoconf wget locales
+update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-5 200
+update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-5 200
+
+# Get clang-format-3.9
+curl -LO https://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz
+curl -LO https://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz.sig
+# Verify the signature.
+gpg --keyserver pool.sks-keyservers.net --recv-keys B6C8F98282B944E3B0D5C2530FC3042E345AD05D
+gpg --verify *.tar.xz.sig
+# Install into /usr/local/.
+tar xJvf *.tar.xz -C /usr/local --strip-components=1
+# Cleanup.
+rm *.tar.xz*
+
+locale-gen en_US.UTF-8
+dpkg-reconfigure locales
+
+# Cleanup.
+rm -rf ~/.ccache ~/.cache
+apt-get autoremove -y
+apt-get clean
+apt-get autoclean
diff --git a/security/nss/automation/taskcluster/docker/setup.sh b/security/nss/automation/taskcluster/docker/setup.sh
index 3ba4e854e..01f9c413a 100644
--- a/security/nss/automation/taskcluster/docker/setup.sh
+++ b/security/nss/automation/taskcluster/docker/setup.sh
@@ -48,8 +48,8 @@ apt-get -y update
apt-get install -y --no-install-recommends ${apt_packages[@]}
# Download clang.
-curl -LO http://releases.llvm.org/4.0.0/clang+llvm-4.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz
-curl -LO http://releases.llvm.org/4.0.0/clang+llvm-4.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz.sig
+curl -LO https://releases.llvm.org/4.0.0/clang+llvm-4.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz
+curl -LO https://releases.llvm.org/4.0.0/clang+llvm-4.0.0-x86_64-linux-gnu-ubuntu-16.04.tar.xz.sig
# Verify the signature.
gpg --keyserver pool.sks-keyservers.net --recv-keys B6C8F98282B944E3B0D5C2530FC3042E345AD05D
gpg --verify *.tar.xz.sig
diff --git a/security/nss/automation/taskcluster/graph/src/context_hash.js b/security/nss/automation/taskcluster/graph/src/context_hash.js
index f0a2e9a88..0699a0590 100644
--- a/security/nss/automation/taskcluster/graph/src/context_hash.js
+++ b/security/nss/automation/taskcluster/graph/src/context_hash.js
@@ -27,14 +27,24 @@ function collectFilesInDirectory(dir) {
});
}
-// Compute a context hash for the given context path.
-export default function (context_path) {
+// A list of hashes for each file in the given path.
+function collectFileHashes(context_path) {
let root = path.join(__dirname, "../../../..");
let dir = path.join(root, context_path);
let files = collectFilesInDirectory(dir).sort();
- let hashes = files.map(file => {
+
+ return files.map(file => {
return sha256(file + "|" + fs.readFileSync(file, "utf-8"));
});
+}
+
+// Compute a context hash for the given context path.
+export default function (context_path) {
+ // Regenerate all images when the image_builder changes.
+ let hashes = collectFileHashes("automation/taskcluster/image_builder");
+
+ // Regenerate images when the image itself changes.
+ hashes = hashes.concat(collectFileHashes(context_path));
// Generate a new prefix every month to ensure the image stays buildable.
let now = new Date();
diff --git a/security/nss/automation/taskcluster/graph/src/extend.js b/security/nss/automation/taskcluster/graph/src/extend.js
index d541a1a3b..90e23ae60 100644
--- a/security/nss/automation/taskcluster/graph/src/extend.js
+++ b/security/nss/automation/taskcluster/graph/src/extend.js
@@ -15,15 +15,29 @@ const LINUX_CLANG39_IMAGE = {
path: "automation/taskcluster/docker-clang-3.9"
};
+const LINUX_GCC44_IMAGE = {
+ name: "linux-gcc-4.4",
+ path: "automation/taskcluster/docker-gcc-4.4"
+};
+
const FUZZ_IMAGE = {
name: "fuzz",
path: "automation/taskcluster/docker-fuzz"
};
+const HACL_GEN_IMAGE = {
+ name: "hacl",
+ path: "automation/taskcluster/docker-hacl"
+};
+
const WINDOWS_CHECKOUT_CMD =
"bash -c \"hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss || " +
"(sleep 2; hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss) || " +
"(sleep 5; hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss)\"";
+const MAC_CHECKOUT_CMD = ["bash", "-c",
+ "hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss || " +
+ "(sleep 2; hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss) || " +
+ "(sleep 5; hg clone -r $NSS_HEAD_REVISION $NSS_HEAD_REPOSITORY nss)"];
/*****************************************************************************/
@@ -51,6 +65,15 @@ queue.filter(task => {
if (task.platform == "aarch64") {
return false;
}
+
+ // No mac
+ if (task.platform == "mac") {
+ return false;
+ }
+ }
+
+ if (task.tests == "fips" && task.platform == "mac") {
+ return false;
}
// Only old make builds have -Ddisable_libpkix=0 and can run chain tests.
@@ -59,8 +82,8 @@ queue.filter(task => {
}
if (task.group == "Test") {
- // Don't run test builds on old make platforms
- if (task.collection == "make") {
+ // Don't run test builds on old make platforms, and not for fips gyp.
+ if (task.collection == "make" || task.collection == "fips") {
return false;
}
}
@@ -78,11 +101,19 @@ queue.filter(task => {
queue.map(task => {
if (task.collection == "asan") {
// CRMF and FIPS tests still leak, unfortunately.
- if (task.tests == "crmf" || task.tests == "fips") {
+ if (task.tests == "crmf") {
task.env.ASAN_OPTIONS = "detect_leaks=0";
}
}
+ // We don't run FIPS SSL tests
+ if (task.tests == "ssl") {
+ if (!task.env) {
+ task.env = {};
+ }
+ task.env.NSS_SSL_TESTS = "crl iopr policy";
+ }
+
// Windows is slow.
if (task.platform == "windows2012-64" && task.tests == "chains") {
task.maxRunTime = 7200;
@@ -128,6 +159,18 @@ export default async function main() {
],
});
+ await scheduleLinux("Linux 64 (opt, make)", {
+ env: {USE_64: "1", BUILD_OPT: "1"},
+ platform: "linux64",
+ image: LINUX_IMAGE,
+ collection: "make",
+ command: [
+ "/bin/bash",
+ "-c",
+ "bin/checkout.sh && nss/automation/taskcluster/scripts/build.sh"
+ ],
+ });
+
await scheduleLinux("Linux 32 (debug, make)", {
platform: "linux32",
image: LINUX_IMAGE,
@@ -153,6 +196,12 @@ export default async function main() {
features: ["allowPtrace"],
}, "--ubsan --asan");
+ await scheduleLinux("Linux 64 (FIPS opt)", {
+ platform: "linux64",
+ collection: "fips",
+ image: LINUX_IMAGE,
+ }, "--enable-fips --opt");
+
await scheduleWindows("Windows 2012 64 (debug, make)", {
platform: "windows2012-64",
collection: "make",
@@ -216,6 +265,70 @@ export default async function main() {
collection: "opt",
}, aarch64_base)
);
+
+ await scheduleMac("Mac (opt)", {collection: "opt"}, "--opt");
+ await scheduleMac("Mac (debug)", {collection: "debug"});
+}
+
+
+async function scheduleMac(name, base, args = "") {
+ let mac_base = merge(base, {
+ env: {
+ PATH: "/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin",
+ NSS_TASKCLUSTER_MAC: "1",
+ DOMSUF: "localdomain",
+ HOST: "localhost",
+ },
+ provisioner: "localprovisioner",
+ workerType: "nss-macos-10-12",
+ platform: "mac"
+ });
+
+ // Build base definition.
+ let build_base = merge({
+ command: [
+ MAC_CHECKOUT_CMD,
+ ["bash", "-c",
+ "nss/automation/taskcluster/scripts/build_gyp.sh", args]
+ ],
+ provisioner: "localprovisioner",
+ workerType: "nss-macos-10-12",
+ platform: "mac",
+ maxRunTime: 7200,
+ artifacts: [{
+ expires: 24 * 7,
+ type: "directory",
+ path: "public"
+ }],
+ kind: "build",
+ symbol: "B"
+ }, mac_base);
+
+ // The task that builds NSPR+NSS.
+ let task_build = queue.scheduleTask(merge(build_base, {name}));
+
+ // The task that generates certificates.
+ let task_cert = queue.scheduleTask(merge(build_base, {
+ name: "Certificates",
+ command: [
+ MAC_CHECKOUT_CMD,
+ ["bash", "-c",
+ "nss/automation/taskcluster/scripts/gen_certs.sh"]
+ ],
+ parent: task_build,
+ symbol: "Certs"
+ }));
+
+ // Schedule tests.
+ scheduleTests(task_build, task_cert, merge(mac_base, {
+ command: [
+ MAC_CHECKOUT_CMD,
+ ["bash", "-c",
+ "nss/automation/taskcluster/scripts/run_tests.sh"]
+ ]
+ }));
+
+ return queue.submit();
}
/*****************************************************************************/
@@ -242,6 +355,45 @@ async function scheduleLinux(name, base, args = "") {
// The task that builds NSPR+NSS.
let task_build = queue.scheduleTask(merge(build_base, {name}));
+ // Make builds run FIPS tests, which need an extra FIPS build.
+ if (base.collection == "make") {
+ let extra_build = queue.scheduleTask(merge(build_base, {
+ env: { NSS_FORCE_FIPS: "1" },
+ group: "FIPS",
+ name: `${name} w/ NSS_FORCE_FIPS`
+ }));
+
+ // The task that generates certificates.
+ let task_cert = queue.scheduleTask(merge(build_base, {
+ name: "Certificates",
+ command: [
+ "/bin/bash",
+ "-c",
+ "bin/checkout.sh && nss/automation/taskcluster/scripts/gen_certs.sh"
+ ],
+ parent: extra_build,
+ symbol: "Certs-F",
+ group: "FIPS",
+ }));
+
+ // Schedule FIPS tests.
+ queue.scheduleTask(merge(base, {
+ parent: task_cert,
+ name: "FIPS",
+ command: [
+ "/bin/bash",
+ "-c",
+ "bin/checkout.sh && nss/automation/taskcluster/scripts/run_tests.sh"
+ ],
+ cycle: "standard",
+ kind: "test",
+ name: "FIPS tests",
+ symbol: "Tests-F",
+ tests: "fips",
+ group: "FIPS"
+ }));
+ }
+
// The task that generates certificates.
let task_cert = queue.scheduleTask(merge(build_base, {
name: "Certificates",
@@ -275,6 +427,26 @@ async function scheduleLinux(name, base, args = "") {
}));
queue.scheduleTask(merge(extra_base, {
+ name: `${name} w/ gcc-4.4`,
+ image: LINUX_GCC44_IMAGE,
+ env: {
+ USE_64: "1",
+ CC: "gcc-4.4",
+ CCC: "g++-4.4",
+ // gcc-4.6 introduced nullptr.
+ NSS_DISABLE_GTESTS: "1",
+ },
+ // Use the old Makefile-based build system, GYP doesn't have a proper GCC
+ // version check for __int128 support. It's mainly meant to cover RHEL6.
+ command: [
+ "/bin/bash",
+ "-c",
+ "bin/checkout.sh && nss/automation/taskcluster/scripts/build.sh",
+ ],
+ symbol: "gcc-4.4"
+ }));
+
+ queue.scheduleTask(merge(extra_base, {
name: `${name} w/ gcc-4.8`,
env: {
CC: "gcc-4.8",
@@ -403,12 +575,13 @@ async function scheduleFuzzing() {
// Schedule MPI fuzzing runs.
let mpi_base = merge(run_base, {group: "MPI"});
- let mpi_names = ["add", "addmod", "div", "expmod", "mod", "mulmod", "sqr",
+ let mpi_names = ["add", "addmod", "div", "mod", "mulmod", "sqr",
"sqrmod", "sub", "submod"];
for (let name of mpi_names) {
scheduleFuzzingRun(mpi_base, `MPI (${name})`, `mpi-${name}`, 4096, name);
}
scheduleFuzzingRun(mpi_base, `MPI (invmod)`, `mpi-invmod`, 256, "invmod");
+ scheduleFuzzingRun(mpi_base, `MPI (expmod)`, `mpi-expmod`, 2048, "expmod");
// Schedule TLS fuzzing runs (non-fuzzing mode).
let tls_base = merge(run_base, {group: "TLS"});
@@ -625,6 +798,43 @@ async function scheduleWindows(name, base, build_script) {
symbol: "B"
});
+ // Make builds run FIPS tests, which need an extra FIPS build.
+ if (base.collection == "make") {
+ let extra_build = queue.scheduleTask(merge(build_base, {
+ env: { NSS_FORCE_FIPS: "1" },
+ group: "FIPS",
+ name: `${name} w/ NSS_FORCE_FIPS`
+ }));
+
+ // The task that generates certificates.
+ let task_cert = queue.scheduleTask(merge(build_base, {
+ name: "Certificates",
+ command: [
+ WINDOWS_CHECKOUT_CMD,
+ "bash -c nss/automation/taskcluster/windows/gen_certs.sh"
+ ],
+ parent: extra_build,
+ symbol: "Certs-F",
+ group: "FIPS",
+ }));
+
+ // Schedule FIPS tests.
+ queue.scheduleTask(merge(base, {
+ parent: task_cert,
+ name: "FIPS",
+ command: [
+ WINDOWS_CHECKOUT_CMD,
+ "bash -c nss/automation/taskcluster/windows/run_tests.sh"
+ ],
+ cycle: "standard",
+ kind: "test",
+ name: "FIPS tests",
+ symbol: "Tests-F",
+ tests: "fips",
+ group: "FIPS"
+ }));
+ }
+
// The task that builds NSPR+NSS.
let task_build = queue.scheduleTask(merge(build_base, {name}));
@@ -703,9 +913,6 @@ function scheduleTests(task_build, task_cert, test_base) {
name: "DB tests", symbol: "DB", tests: "dbtests"
}));
queue.scheduleTask(merge(cert_base, {
- name: "FIPS tests", symbol: "FIPS", tests: "fips"
- }));
- queue.scheduleTask(merge(cert_base, {
name: "Merge tests", symbol: "Merge", tests: "merge"
}));
queue.scheduleTask(merge(cert_base, {
@@ -773,5 +980,16 @@ async function scheduleTools() {
]
}));
+ queue.scheduleTask(merge(base, {
+ symbol: "hacl",
+ name: "hacl",
+ image: HACL_GEN_IMAGE,
+ command: [
+ "/bin/bash",
+ "-c",
+ "bin/checkout.sh && nss/automation/taskcluster/scripts/run_hacl.sh"
+ ]
+ }));
+
return queue.submit();
}
diff --git a/security/nss/automation/taskcluster/graph/src/image_builder.js b/security/nss/automation/taskcluster/graph/src/image_builder.js
index bc90e0242..b89b6980c 100644
--- a/security/nss/automation/taskcluster/graph/src/image_builder.js
+++ b/security/nss/automation/taskcluster/graph/src/image_builder.js
@@ -31,13 +31,11 @@ export async function buildTask({name, path}) {
return {
name: "Image Builder",
- image: "taskcluster/image_builder:0.1.5",
+ image: "nssdev/image_builder:0.1.5",
routes: ["index." + ns],
env: {
- HEAD_REPOSITORY: process.env.NSS_HEAD_REPOSITORY,
- BASE_REPOSITORY: process.env.NSS_HEAD_REPOSITORY,
- HEAD_REV: process.env.NSS_HEAD_REVISION,
- HEAD_REF: process.env.NSS_HEAD_REVISION,
+ NSS_HEAD_REPOSITORY: process.env.NSS_HEAD_REPOSITORY,
+ NSS_HEAD_REVISION: process.env.NSS_HEAD_REVISION,
PROJECT: process.env.TC_PROJECT,
CONTEXT_PATH: path,
HASH: hash
@@ -52,10 +50,11 @@ export async function buildTask({name, path}) {
command: [
"/bin/bash",
"-c",
- "/home/worker/bin/build_image.sh"
+ "bin/checkout.sh && nss/automation/taskcluster/scripts/build_image.sh"
],
platform: "nss-decision",
features: ["dind"],
+ maxRunTime: 7200,
kind: "build",
symbol: "I"
};
diff --git a/security/nss/automation/taskcluster/graph/src/try_syntax.js b/security/nss/automation/taskcluster/graph/src/try_syntax.js
index 7748e068a..1f4e12eee 100644
--- a/security/nss/automation/taskcluster/graph/src/try_syntax.js
+++ b/security/nss/automation/taskcluster/graph/src/try_syntax.js
@@ -22,10 +22,10 @@ function parseOptions(opts) {
}
// Parse platforms.
- let allPlatforms = ["linux", "linux64", "linux64-asan",
+ let allPlatforms = ["linux", "linux64", "linux64-asan", "linux64-fips",
"win", "win64", "win-make", "win64-make",
"linux64-make", "linux-make", "linux-fuzz",
- "linux64-fuzz", "aarch64"];
+ "linux64-fuzz", "aarch64", "mac"];
let platforms = intersect(opts.platform.split(/\s*,\s*/), allPlatforms);
// If the given value is nonsense or "none" default to all platforms.
@@ -51,7 +51,7 @@ function parseOptions(opts) {
}
// Parse tools.
- let allTools = ["clang-format", "scan-build"];
+ let allTools = ["clang-format", "scan-build", "hacl"];
let tools = intersect(opts.tools.split(/\s*,\s*/), allTools);
// If the given value is "all" run all tools.
@@ -111,6 +111,7 @@ function filter(opts) {
"linux": "linux32",
"linux-fuzz": "linux32",
"linux64-asan": "linux64",
+ "linux64-fips": "linux64",
"linux64-fuzz": "linux64",
"linux64-make": "linux64",
"linux-make": "linux32",
@@ -126,6 +127,8 @@ function filter(opts) {
// Additional checks.
if (platform == "linux64-asan") {
keep &= coll("asan");
+ } else if (platform == "linux64-fips") {
+ keep &= coll("fips");
} else if (platform == "linux64-make" || platform == "linux-make" ||
platform == "win64-make" || platform == "win-make") {
keep &= coll("make");
diff --git a/security/nss/automation/taskcluster/image_builder/Dockerfile b/security/nss/automation/taskcluster/image_builder/Dockerfile
new file mode 100644
index 000000000..f8b4edcc5
--- /dev/null
+++ b/security/nss/automation/taskcluster/image_builder/Dockerfile
@@ -0,0 +1,23 @@
+FROM ubuntu:16.04
+MAINTAINER Tim Taubert <ttaubert@mozilla.com>
+
+WORKDIR /home/worker
+
+ENV DEBIAN_FRONTEND noninteractive
+
+RUN apt-get update && apt-get install -y apt-transport-https apt-utils
+RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9 && \
+ sh -c "echo deb https://get.docker.io/ubuntu docker main \
+ > /etc/apt/sources.list.d/docker.list"
+RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 41BD8711B1F0EC2B0D85B91CF59CE3A8323293EE && \
+ sh -c "echo deb http://ppa.launchpad.net/mercurial-ppa/releases/ubuntu xenial main \
+ > /etc/apt/sources.list.d/mercurial.list"
+RUN apt-get update && apt-get install -y \
+ lxc-docker-1.6.1 \
+ mercurial
+
+ADD bin /home/worker/bin
+RUN chmod +x /home/worker/bin/*
+
+# Set a default command useful for debugging
+CMD ["/bin/bash", "--login"]
diff --git a/security/nss/automation/taskcluster/image_builder/VERSION b/security/nss/automation/taskcluster/image_builder/VERSION
new file mode 100644
index 000000000..9faa1b7a7
--- /dev/null
+++ b/security/nss/automation/taskcluster/image_builder/VERSION
@@ -0,0 +1 @@
+0.1.5
diff --git a/security/nss/automation/taskcluster/image_builder/bin/checkout.sh b/security/nss/automation/taskcluster/image_builder/bin/checkout.sh
new file mode 100644
index 000000000..0cdd2ac40
--- /dev/null
+++ b/security/nss/automation/taskcluster/image_builder/bin/checkout.sh
@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -v -e -x
+
+# Default values for testing.
+REVISION=${NSS_HEAD_REVISION:-default}
+REPOSITORY=${NSS_HEAD_REPOSITORY:-https://hg.mozilla.org/projects/nss}
+
+# Clone NSS.
+for i in 0 2 5; do
+ sleep $i
+ hg clone -r $REVISION $REPOSITORY nss && exit 0
+ rm -rf nss
+done
+exit 1
diff --git a/security/nss/automation/taskcluster/scripts/build_gyp.sh b/security/nss/automation/taskcluster/scripts/build_gyp.sh
index 7190bd5c4..fb3a33a52 100755
--- a/security/nss/automation/taskcluster/scripts/build_gyp.sh
+++ b/security/nss/automation/taskcluster/scripts/build_gyp.sh
@@ -9,5 +9,10 @@ hg_clone https://hg.mozilla.org/projects/nspr ./nspr default
nss/build.sh -g -v "$@"
# Package.
-mkdir artifacts
-tar cvfjh artifacts/dist.tar.bz2 dist
+if [[ $(uname) = "Darwin" ]]; then
+ mkdir -p public
+ tar cvfjh public/dist.tar.bz2 dist
+else
+ mkdir artifacts
+ tar cvfjh artifacts/dist.tar.bz2 dist
+fi
diff --git a/security/nss/automation/taskcluster/scripts/build_image.sh b/security/nss/automation/taskcluster/scripts/build_image.sh
new file mode 100644
index 000000000..b422214e7
--- /dev/null
+++ b/security/nss/automation/taskcluster/scripts/build_image.sh
@@ -0,0 +1,24 @@
+#!/bin/bash -vex
+
+set -x -e -v
+
+# Prefix errors with taskcluster error prefix so that they are parsed by Treeherder
+raise_error() {
+ echo
+ echo "[taskcluster-image-build:error] $1"
+ exit 1
+}
+
+# Ensure that the PROJECT is specified so the image can be indexed
+test -n "$PROJECT" || raise_error "Project must be provided."
+test -n "$HASH" || raise_error "Context Hash must be provided."
+
+CONTEXT_PATH=/home/worker/nss/$CONTEXT_PATH
+
+test -d $CONTEXT_PATH || raise_error "Context Path $CONTEXT_PATH does not exist."
+test -f "$CONTEXT_PATH/Dockerfile" || raise_error "Dockerfile must be present in $CONTEXT_PATH."
+
+docker build -t $PROJECT:$HASH $CONTEXT_PATH
+
+mkdir /artifacts
+docker save $PROJECT:$HASH > /artifacts/image.tar
diff --git a/security/nss/automation/taskcluster/scripts/gen_certs.sh b/security/nss/automation/taskcluster/scripts/gen_certs.sh
index b8d4f60ba..c03db7e9c 100755
--- a/security/nss/automation/taskcluster/scripts/gen_certs.sh
+++ b/security/nss/automation/taskcluster/scripts/gen_certs.sh
@@ -12,5 +12,10 @@ NSS_TESTS=cert NSS_CYCLES="standard pkix sharedb" $(dirname $0)/run_tests.sh
echo 1 > tests_results/security/localhost
# Package.
-mkdir artifacts
-tar cvfjh artifacts/dist.tar.bz2 dist tests_results
+if [[ $(uname) = "Darwin" ]]; then
+ mkdir -p public
+ tar cvfjh public/dist.tar.bz2 dist tests_results
+else
+ mkdir artifacts
+ tar cvfjh artifacts/dist.tar.bz2 dist tests_results
+fi
diff --git a/security/nss/automation/taskcluster/scripts/run_hacl.sh b/security/nss/automation/taskcluster/scripts/run_hacl.sh
new file mode 100644
index 000000000..281075eef
--- /dev/null
+++ b/security/nss/automation/taskcluster/scripts/run_hacl.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+if [[ $(id -u) -eq 0 ]]; then
+ # Drop privileges by re-running this script.
+ # Note: this mangles arguments, better to avoid running scripts as root.
+ exec su worker -c "$0 $*"
+fi
+
+set -e -x -v
+
+# The docker image this is running in has the HACL* and NSS sources.
+# The extracted C code from HACL* is already generated and the HACL* tests were
+# successfully executed.
+
+# Verify Poly1305 (doesn't work in docker image build)
+make verify -C ~/hacl-star/code/poly1305 -j$(nproc)
+
+# Add license header to specs
+spec_files=($(find ~/hacl-star/specs -type f -name '*.fst'))
+for f in "${spec_files[@]}"; do
+ cat /tmp/license.txt "$f" > /tmp/tmpfile && mv /tmp/tmpfile "$f"
+done
+
+# Format the extracted C code.
+cd ~/hacl-star/snapshots/nss
+cp ~/nss/.clang-format .
+find . -type f -name '*.[ch]' -exec clang-format -i {} \+
+
+# These diff commands will return 1 if there are differences and stop the script.
+files=($(find ~/nss/lib/freebl/verified/ -type f -name '*.[ch]'))
+for f in "${files[@]}"; do
+ diff $f $(basename "$f")
+done
+
+# Check that the specs didn't change either.
+cd ~/hacl-star/specs
+files=($(find ~/nss/lib/freebl/verified/specs -type f))
+for f in "${files[@]}"; do
+ diff $f $(basename "$f")
+done
diff --git a/security/nss/automation/taskcluster/scripts/split.sh b/security/nss/automation/taskcluster/scripts/split.sh
index 4d18385ec..fded64e1b 100644
--- a/security/nss/automation/taskcluster/scripts/split.sh
+++ b/security/nss/automation/taskcluster/scripts/split.sh
@@ -23,16 +23,10 @@ split_util() {
# Copy everything.
cp -R $nssdir $dstdir
- # Skip gtests when building.
- sed '/^DIRS = /s/ cpputil gtests$//' $nssdir/manifest.mn > $dstdir/manifest.mn-t && mv $dstdir/manifest.mn-t $dstdir/manifest.mn
-
# Remove subdirectories that we don't want.
rm -rf $dstdir/cmd
- rm -rf $dstdir/tests
rm -rf $dstdir/lib
rm -rf $dstdir/automation
- rm -rf $dstdir/gtests
- rm -rf $dstdir/cpputil
rm -rf $dstdir/doc
# Start with an empty cmd lib directories to be filled selectively.
diff --git a/security/nss/automation/taskcluster/windows/releng.manifest b/security/nss/automation/taskcluster/windows/releng.manifest
index 68d2c1d9e..d571c544d 100644
--- a/security/nss/automation/taskcluster/windows/releng.manifest
+++ b/security/nss/automation/taskcluster/windows/releng.manifest
@@ -1,10 +1,10 @@
[
{
- "version": "Visual Studio 2015 Update 3 14.0.25425.01 / SDK 10.0.14393.0",
- "size": 326656969,
- "digest": "babc414ffc0457d27f5a1ed24a8e4873afbe2f1c1a4075469a27c005e1babc3b2a788f643f825efedff95b79686664c67ec4340ed535487168a3482e68559bc7",
+ "version": "Visual Studio 2017 15.4.2 / SDK 10.0.15063.0",
+ "size": 303146863,
+ "digest": "18700889e6b5e81613b9cf57ce4e0d46a6ee45bb4c5c33bae2604a5275326128775b8a032a1eb178c5db973746d565340c4e36d98375789e1d5bd836ab16ba58",
"algorithm": "sha512",
- "filename": "vs2015u3.zip",
+ "filename": "vs2017_15.4.2.zip",
"unpack": true
},
{
diff --git a/security/nss/automation/taskcluster/windows/setup.sh b/security/nss/automation/taskcluster/windows/setup.sh
index 7def50db4..36a040ba1 100644
--- a/security/nss/automation/taskcluster/windows/setup.sh
+++ b/security/nss/automation/taskcluster/windows/setup.sh
@@ -2,12 +2,12 @@
set -v -e -x
-export VSPATH="$(pwd)/vs2015u3"
+export VSPATH="$(pwd)/vs2017_15.4.2"
export NINJA_PATH="$(pwd)/ninja/bin"
export WINDOWSSDKDIR="${VSPATH}/SDK"
export VS90COMNTOOLS="${VSPATH}/VC"
-export INCLUDE="${VSPATH}/VC/include:${VSPATH}/SDK/Include/10.0.14393.0/ucrt:${VSPATH}/SDK/Include/10.0.14393.0/shared:${VSPATH}/SDK/Include/10.0.14393.0/um"
+export INCLUDE="${VSPATH}/VC/include:${VSPATH}/SDK/Include/10.0.15063.0/ucrt:${VSPATH}/SDK/Include/10.0.15063.0/shared:${VSPATH}/SDK/Include/10.0.15063.0/um"
# Usage: hg_clone repo dir [revision=@]
hg_clone() {
@@ -23,4 +23,4 @@ hg_clone() {
}
hg_clone https://hg.mozilla.org/build/tools tools default
-tools/scripts/tooltool/tooltool_wrapper.sh $(dirname $0)/releng.manifest https://api.pub.build.mozilla.org/tooltool/ non-existant-file.sh /c/mozilla-build/python/python.exe /c/builds/tooltool.py --authentication-file /c/builds/relengapi.tok -c /c/builds/tooltool_cache
+tools/scripts/tooltool/tooltool_wrapper.sh $(dirname $0)/releng.manifest https://tooltool.mozilla-releng.net/ non-existant-file.sh /c/mozilla-build/python/python.exe /c/builds/tooltool.py --authentication-file /c/builds/relengapi.tok -c /c/builds/tooltool_cache
diff --git a/security/nss/automation/taskcluster/windows/setup32.sh b/security/nss/automation/taskcluster/windows/setup32.sh
index bcddabfa3..19bed284d 100644
--- a/security/nss/automation/taskcluster/windows/setup32.sh
+++ b/security/nss/automation/taskcluster/windows/setup32.sh
@@ -4,7 +4,7 @@ set -v -e -x
source $(dirname $0)/setup.sh
-export WIN32_REDIST_DIR="${VSPATH}/VC/redist/x86/Microsoft.VC140.CRT"
+export WIN32_REDIST_DIR="${VSPATH}/VC/redist/x86/Microsoft.VC141.CRT"
export WIN_UCRT_REDIST_DIR="${VSPATH}/SDK/Redist/ucrt/DLLs/x86"
-export PATH="${NINJA_PATH}:${VSPATH}/VC/bin/amd64_x86:${VSPATH}/VC/bin/amd64:${VSPATH}/VC/bin:${VSPATH}/SDK/bin/x86:${VSPATH}/SDK/bin/x64:${VSPATH}/VC/redist/x86/Microsoft.VC140.CRT:${VSPATH}/VC/redist/x64/Microsoft.VC140.CRT:${VSPATH}/SDK/Redist/ucrt/DLLs/x86:${VSPATH}/SDK/Redist/ucrt/DLLs/x64:${PATH}"
-export LIB="${VSPATH}/VC/lib:${VSPATH}/SDK/lib/10.0.14393.0/ucrt/x86:${VSPATH}/SDK/lib/10.0.14393.0/um/x86"
+export PATH="${NINJA_PATH}:${VSPATH}/VC/bin/Hostx64/x86:${VSPATH}/VC/bin/Hostx64/x64:${VSPATH}/VC/Hostx86/x86:${VSPATH}/SDK/bin/10.0.15063.0/x64:${VSPATH}/VC/redist/x86/Microsoft.VC141.CRT:${VSPATH}/SDK/Redist/ucrt/DLLs/x86:${PATH}"
+export LIB="${VSPATH}/VC/lib/x86:${VSPATH}/SDK/lib/10.0.15063.0/ucrt/x86:${VSPATH}/SDK/lib/10.0.15063.0/um/x86"
diff --git a/security/nss/automation/taskcluster/windows/setup64.sh b/security/nss/automation/taskcluster/windows/setup64.sh
index f308298c1..d16cb0ec9 100644
--- a/security/nss/automation/taskcluster/windows/setup64.sh
+++ b/security/nss/automation/taskcluster/windows/setup64.sh
@@ -4,7 +4,7 @@ set -v -e -x
source $(dirname $0)/setup.sh
-export WIN32_REDIST_DIR="${VSPATH}/VC/redist/x64/Microsoft.VC140.CRT"
+export WIN32_REDIST_DIR="${VSPATH}/VC/redist/x64/Microsoft.VC141.CRT"
export WIN_UCRT_REDIST_DIR="${VSPATH}/SDK/Redist/ucrt/DLLs/x64"
-export PATH="${NINJA_PATH}:${VSPATH}/VC/bin/amd64:${VSPATH}/VC/bin:${VSPATH}/SDK/bin/x64:${VSPATH}/VC/redist/x64/Microsoft.VC140.CRT:${VSPATH}/SDK/Redist/ucrt/DLLs/x64:${PATH}"
-export LIB="${VSPATH}/VC/lib/amd64:${VSPATH}/SDK/lib/10.0.14393.0/ucrt/x64:${VSPATH}/SDK/lib/10.0.14393.0/um/x64"
+export PATH="${NINJA_PATH}:${VSPATH}/VC/bin/Hostx64/x64:${VSPATH}/VC/bin/Hostx86/x86:${VSPATH}/SDK/bin/10.0.15063.0/x64:${VSPATH}/VC/redist/x64/Microsoft.VC141.CRT:${VSPATH}/SDK/Redist/ucrt/DLLs/x64:${PATH}"
+export LIB="${VSPATH}/VC/lib/x64:${VSPATH}/SDK/lib/10.0.15063.0/ucrt/x64:${VSPATH}/SDK/lib/10.0.15063.0/um/x64"