From f017b749ea9f1586d2308504553d40bf4cc5439d Mon Sep 17 00:00:00 2001 From: wolfbeast Date: Tue, 6 Feb 2018 11:46:26 +0100 Subject: Update NSS to 3.32.1-RTM --- .../nss/automation/taskcluster/scripts/build.sh | 13 +- .../automation/taskcluster/scripts/build_gyp.sh | 11 +- .../automation/taskcluster/scripts/build_nspr.sh | 18 +++ .../automation/taskcluster/scripts/build_nss.sh | 39 ++++++ .../taskcluster/scripts/build_softoken.sh | 30 ++++ .../automation/taskcluster/scripts/build_util.sh | 25 ++++ .../taskcluster/scripts/extend_task_graph.sh | 7 +- .../nss/automation/taskcluster/scripts/fuzz.sh | 26 +++- .../automation/taskcluster/scripts/gen_certs.sh | 12 +- .../taskcluster/scripts/run_clang_format.sh | 63 --------- .../taskcluster/scripts/run_scan_build.sh | 9 +- .../automation/taskcluster/scripts/run_tests.sh | 10 +- .../nss/automation/taskcluster/scripts/split.sh | 154 +++++++++++++++++++++ .../nss/automation/taskcluster/scripts/tools.sh | 10 ++ 14 files changed, 311 insertions(+), 116 deletions(-) create mode 100644 security/nss/automation/taskcluster/scripts/build_nspr.sh create mode 100644 security/nss/automation/taskcluster/scripts/build_nss.sh create mode 100644 security/nss/automation/taskcluster/scripts/build_softoken.sh create mode 100644 security/nss/automation/taskcluster/scripts/build_util.sh delete mode 100755 security/nss/automation/taskcluster/scripts/run_clang_format.sh create mode 100644 security/nss/automation/taskcluster/scripts/split.sh (limited to 'security/nss/automation/taskcluster/scripts') diff --git a/security/nss/automation/taskcluster/scripts/build.sh b/security/nss/automation/taskcluster/scripts/build.sh index 69968b138..649fdaa1b 100755 --- a/security/nss/automation/taskcluster/scripts/build.sh +++ b/security/nss/automation/taskcluster/scripts/build.sh @@ -1,14 +1,17 @@ #!/usr/bin/env bash -source $(dirname $0)/tools.sh +source $(dirname "$0")/tools.sh -if [[ $(id -u) -eq 0 ]]; then - # Drop privileges by re-running this script. - exec su worker $0 +if [ -n "$NSS_BUILD_MODULAR" ]; then + $(dirname "$0")/build_nspr.sh || exit $? + $(dirname "$0")/build_util.sh || exit $? + $(dirname "$0")/build_softoken.sh || exit $? + $(dirname "$0")/build_nss.sh || exit $? + exit fi # Clone NSPR if needed. -hg_clone https://hg.mozilla.org/projects/nspr nspr default +hg_clone https://hg.mozilla.org/projects/nspr ./nspr default # Build. make -C nss nss_build_all diff --git a/security/nss/automation/taskcluster/scripts/build_gyp.sh b/security/nss/automation/taskcluster/scripts/build_gyp.sh index 590e634a3..7190bd5c4 100755 --- a/security/nss/automation/taskcluster/scripts/build_gyp.sh +++ b/security/nss/automation/taskcluster/scripts/build_gyp.sh @@ -1,17 +1,12 @@ #!/usr/bin/env bash -source $(dirname $0)/tools.sh - -if [[ $(id -u) -eq 0 ]]; then - # Drop privileges by re-running this script. - exec su worker -c "$0 $*" -fi +source $(dirname "$0")/tools.sh # Clone NSPR if needed. -hg_clone https://hg.mozilla.org/projects/nspr nspr default +hg_clone https://hg.mozilla.org/projects/nspr ./nspr default # Build. -nss/build.sh ${*--g -v} +nss/build.sh -g -v "$@" # Package. mkdir artifacts diff --git a/security/nss/automation/taskcluster/scripts/build_nspr.sh b/security/nss/automation/taskcluster/scripts/build_nspr.sh new file mode 100644 index 000000000..4d1903460 --- /dev/null +++ b/security/nss/automation/taskcluster/scripts/build_nspr.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -v -e -x + +source $(dirname $0)/tools.sh + +# Clone NSPR if needed. +hg_clone https://hg.mozilla.org/projects/nspr nspr default + +# Build. +rm -rf dist +make -C nss build_nspr + +# Package. +test -d artifacts || mkdir artifacts +rm -rf dist-nspr +mv dist dist-nspr +tar cvfjh artifacts/dist-nspr.tar.bz2 dist-nspr diff --git a/security/nss/automation/taskcluster/scripts/build_nss.sh b/security/nss/automation/taskcluster/scripts/build_nss.sh new file mode 100644 index 000000000..b909bc30e --- /dev/null +++ b/security/nss/automation/taskcluster/scripts/build_nss.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +set -v -e -x + +source $(dirname $0)/tools.sh +source $(dirname $0)/split.sh + +test -d dist-softoken || { echo "run build_softoken.sh first" 1>&2; exit 1; } + +rm -rf nss-nss +split_nss nss nss-nss + +# Build. +export NSS_BUILD_WITHOUT_SOFTOKEN=1 +export NSS_USE_SYSTEM_FREEBL=1 + +platform=`make -s -C nss platform` + +export NSPR_LIB_DIR="$PWD/dist-nspr/$platform/lib" +export NSSUTIL_LIB_DIR="$PWD/dist-util/$platform/lib" +export FREEBL_LIB_DIR="$PWD/dist-softoken/$platform/lib" +export SOFTOKEN_LIB_DIR="$PWD/dist-softoken/$platform/lib" +export FREEBL_LIBS=-lfreebl + +export NSS_NO_PKCS11_BYPASS=1 +export FREEBL_NO_DEPEND=1 + +export LIBRARY_PATH="$PWD/dist-nspr/$platform/lib:$PWD/dist-util/$platform/lib:$PWD/dist-softoken/$platform/lib" +export LD_LIBRARY_PATH="$LIBRARY_PATH:$LD_LIBRARY_PATH" +export INCLUDES="-I$PWD/dist-nspr/$platform/include -I$PWD/dist-util/public/nss -I$PWD/dist-softoken/public/nss" + +rm -rf dist +make -C nss-nss nss_build_all + +# Package. +test -d artifacts || mkdir artifacts +rm -rf dist-nss +mv dist dist-nss +tar cvfjh artifacts/dist-nss.tar.bz2 dist-nss diff --git a/security/nss/automation/taskcluster/scripts/build_softoken.sh b/security/nss/automation/taskcluster/scripts/build_softoken.sh new file mode 100644 index 000000000..e5aaecccf --- /dev/null +++ b/security/nss/automation/taskcluster/scripts/build_softoken.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +set -v -e -x + +source $(dirname $0)/tools.sh +source $(dirname $0)/split.sh + +test -d dist-util || { echo "run build_util.sh first" 1>&2; exit 1; } + +rm -rf nss-softoken +split_softoken nss nss-softoken + +# Build. +platform=`make -s -C nss platform` +export LIBRARY_PATH="$PWD/dist-nspr/$platform/lib:$PWD/dist-util/$platform/lib" +export LD_LIBRARY_PATH="$LIBRARY_PATH:$LD_LIBRARY_PATH" +export INCLUDES="-I$PWD/dist-nspr/$platform/include -I$PWD/dist-util/public/nss" +export NSS_BUILD_SOFTOKEN_ONLY=1 + +rm -rf dist +make -C nss-softoken nss_build_all + +mv dist/private/nss/blapi.h dist/public/nss +mv dist/private/nss/alghmac.h dist/public/nss + +# Package. +test -d artifacts || mkdir artifacts +rm -rf dist-softoken +mv dist dist-softoken +tar cvfjh artifacts/dist-softoken.tar.bz2 dist-softoken diff --git a/security/nss/automation/taskcluster/scripts/build_util.sh b/security/nss/automation/taskcluster/scripts/build_util.sh new file mode 100644 index 000000000..0d2ecc5e8 --- /dev/null +++ b/security/nss/automation/taskcluster/scripts/build_util.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -v -e -x + +source $(dirname $0)/tools.sh +source $(dirname $0)/split.sh + +rm -rf nss-util +split_util nss nss-util + +# Build. +platform=`make -s -C nss platform` +export LIBRARY_PATH="$PWD/dist-nspr/$platform/lib" +export LD_LIBRARY_PATH="$LIBRARY_PATH:$LD_LIBRARY_PATH" +export INCLUDES="-I$PWD/dist-nspr/$platform/include" +export NSS_BUILD_UTIL_ONLY=1 + +rm -rf dist +make -C nss-util nss_build_all + +# Package. +test -d artifacts || mkdir artifacts +rm -rf dist-util +mv dist dist-util +tar cvfjh artifacts/dist-util.tar.bz2 dist-util diff --git a/security/nss/automation/taskcluster/scripts/extend_task_graph.sh b/security/nss/automation/taskcluster/scripts/extend_task_graph.sh index 5a3fb8d98..ade84cd60 100755 --- a/security/nss/automation/taskcluster/scripts/extend_task_graph.sh +++ b/security/nss/automation/taskcluster/scripts/extend_task_graph.sh @@ -1,11 +1,6 @@ #!/usr/bin/env bash -set -v -e -x - -if [ $(id -u) = 0 ]; then - # Drop privileges by re-running this script. - exec su worker $0 -fi +source $(dirname "$0")/tools.sh mkdir -p /home/worker/artifacts diff --git a/security/nss/automation/taskcluster/scripts/fuzz.sh b/security/nss/automation/taskcluster/scripts/fuzz.sh index 5f8dd7bff..75851ff5b 100755 --- a/security/nss/automation/taskcluster/scripts/fuzz.sh +++ b/security/nss/automation/taskcluster/scripts/fuzz.sh @@ -1,20 +1,32 @@ #!/usr/bin/env bash -source $(dirname $0)/tools.sh +source $(dirname "$0")/tools.sh -if [ $(id -u) = 0 ]; then - # Drop privileges by re-running this script. - exec su worker -c "$0 $*" -fi +type="$1" +shift # Fetch artifact if needed. fetch_dist # Clone corpus. -./nss/fuzz/clone_corpus.sh +./nss/fuzz/config/clone_corpus.sh + +# Ensure we have a corpus. +if [ ! -d "nss/fuzz/corpus/$type" ]; then + mkdir -p nss/fuzz/corpus/$type + + set +x + + # Create a corpus out of what we have. + for f in $(find nss/fuzz/corpus -type f); do + cp $f "nss/fuzz/corpus/$type" + done + + set -x +fi # Fetch objdir name. objdir=$(cat dist/latest) # Run nssfuzz. -LD_LIBRARY_PATH=$LD_LIBRARY_PATH:dist/$objdir/lib dist/$objdir/bin/nssfuzz $* +dist/$objdir/bin/nssfuzz-"$type" "$@" diff --git a/security/nss/automation/taskcluster/scripts/gen_certs.sh b/security/nss/automation/taskcluster/scripts/gen_certs.sh index aee100147..b8d4f60ba 100755 --- a/security/nss/automation/taskcluster/scripts/gen_certs.sh +++ b/security/nss/automation/taskcluster/scripts/gen_certs.sh @@ -1,16 +1,6 @@ #!/usr/bin/env bash -set -v -e -x - -source $(dirname $0)/tools.sh - -if [ $(id -u) = 0 ]; then - # Stupid Docker. - echo "127.0.0.1 localhost.localdomain" >> /etc/hosts - - # Drop privileges by re-running this script. - exec su worker $0 -fi +source $(dirname "$0")/tools.sh # Fetch artifact if needed. fetch_dist diff --git a/security/nss/automation/taskcluster/scripts/run_clang_format.sh b/security/nss/automation/taskcluster/scripts/run_clang_format.sh deleted file mode 100755 index c4b60290f..000000000 --- a/security/nss/automation/taskcluster/scripts/run_clang_format.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/usr/bin/env bash - -set -v -e -x - -if [ $(id -u) -eq 0 ]; then - # Drop privileges by re-running this script. - exec su worker $0 "$@" -fi - -# Apply clang-format on the provided folder and verify that this doesn't change any file. -# If any file differs after formatting, the script eventually exits with 1. -# Any differences between formatted and unformatted files is printed to stdout to give a hint what's wrong. - -# Includes a default set of directories. - -if [ $# -gt 0 ]; then - dirs=("$@") -else - top=$(dirname $0)/../../.. - dirs=( \ - "$top/cmd" \ - "$top/fuzz" \ - "$top/lib/base" \ - "$top/lib/certdb" \ - "$top/lib/certhigh" \ - "$top/lib/ckfw" \ - "$top/lib/crmf" \ - "$top/lib/cryptohi" \ - "$top/lib/dbm" \ - "$top/lib/dev" \ - "$top/lib/freebl" \ - "$top/lib/jar" \ - "$top/lib/nss" \ - "$top/lib/pk11wrap" \ - "$top/lib/pkcs7" \ - "$top/lib/pkcs12" \ - "$top/lib/pki" \ - "$top/lib/smime" \ - "$top/lib/softoken" \ - "$top/lib/ssl" \ - "$top/lib/sysinit" \ - "$top/lib/util" \ - "$top/gtests/common" \ - "$top/gtests/der_gtest" \ - "$top/gtests/freebl_gtest" \ - "$top/gtests/pk11_gtest" \ - "$top/gtests/ssl_gtest" \ - "$top/gtests/util_gtest" \ - ) -fi - -for dir in "${dirs[@]}"; do - find "$dir" -type f \( -name '*.[ch]' -o -name '*.cc' \) -exec clang-format -i {} \+ -done - -TMPFILE=$(mktemp /tmp/$(basename $0).XXXXXX) -trap 'rm $TMPFILE' exit -if (cd $(dirname $0); hg root >/dev/null 2>&1); then - hg diff --git "$top" | tee $TMPFILE -else - git -C "$top" diff | tee $TMPFILE -fi -[[ ! -s $TMPFILE ]] diff --git a/security/nss/automation/taskcluster/scripts/run_scan_build.sh b/security/nss/automation/taskcluster/scripts/run_scan_build.sh index 99f80ab5f..4024c226e 100755 --- a/security/nss/automation/taskcluster/scripts/run_scan_build.sh +++ b/security/nss/automation/taskcluster/scripts/run_scan_build.sh @@ -1,15 +1,10 @@ #!/usr/bin/env bash -source $(dirname $0)/tools.sh - -if [ $(id -u) = 0 ]; then - # Drop privileges by re-running this script. - exec su worker $0 $@ -fi +source $(dirname "$0")/tools.sh # Clone NSPR if needed. if [ ! -d "nspr" ]; then - hg_clone https://hg.mozilla.org/projects/nspr nspr default + hg_clone https://hg.mozilla.org/projects/nspr ./nspr default fi # Build. diff --git a/security/nss/automation/taskcluster/scripts/run_tests.sh b/security/nss/automation/taskcluster/scripts/run_tests.sh index 4c87e7e32..b8e26761a 100755 --- a/security/nss/automation/taskcluster/scripts/run_tests.sh +++ b/security/nss/automation/taskcluster/scripts/run_tests.sh @@ -1,14 +1,6 @@ #!/usr/bin/env bash -source $(dirname $0)/tools.sh - -if [ $(id -u) = 0 ]; then - # Stupid Docker. - echo "127.0.0.1 localhost.localdomain" >> /etc/hosts - - # Drop privileges by re-running this script. - exec su worker $0 -fi +source $(dirname "$0")/tools.sh # Fetch artifact if needed. fetch_dist diff --git a/security/nss/automation/taskcluster/scripts/split.sh b/security/nss/automation/taskcluster/scripts/split.sh new file mode 100644 index 000000000..4d18385ec --- /dev/null +++ b/security/nss/automation/taskcluster/scripts/split.sh @@ -0,0 +1,154 @@ +copy_top() +{ + srcdir_="$1" + dstdir_="$2" + files=`find "$srcdir_" -maxdepth 1 -mindepth 1 -type f` + for f in $files; do + cp -p "$f" "$dstdir_" + done +} + +split_util() { + nssdir="$1" + dstdir="$2" + + # Prepare a source tree only containing files to build nss-util: + # + # nss/dbm full directory + # nss/coreconf full directory + # nss top files only + # nss/lib top files only + # nss/lib/util full directory + + # Copy everything. + cp -R $nssdir $dstdir + + # Skip gtests when building. + sed '/^DIRS = /s/ cpputil gtests$//' $nssdir/manifest.mn > $dstdir/manifest.mn-t && mv $dstdir/manifest.mn-t $dstdir/manifest.mn + + # Remove subdirectories that we don't want. + rm -rf $dstdir/cmd + rm -rf $dstdir/tests + rm -rf $dstdir/lib + rm -rf $dstdir/automation + rm -rf $dstdir/gtests + rm -rf $dstdir/cpputil + rm -rf $dstdir/doc + + # Start with an empty cmd lib directories to be filled selectively. + mkdir $dstdir/cmd + cp $nssdir/cmd/Makefile $dstdir/cmd + cp $nssdir/cmd/manifest.mn $dstdir/cmd + cp $nssdir/cmd/platlibs.mk $dstdir/cmd + cp $nssdir/cmd/platrules.mk $dstdir/cmd + + # Copy some files at the top and the util subdirectory recursively. + mkdir $dstdir/lib + cp $nssdir/lib/Makefile $dstdir/lib + cp $nssdir/lib/manifest.mn $dstdir/lib + cp -R $nssdir/lib/util $dstdir/lib/util +} + +split_softoken() { + nssdir="$1" + dstdir="$2" + + # Prepare a source tree only containing files to build nss-softoken: + # + # nss/dbm full directory + # nss/coreconf full directory + # nss top files only + # nss/lib top files only + # nss/lib/freebl full directory + # nss/lib/softoken full directory + # nss/lib/softoken/dbm full directory + + # Copy everything. + cp -R $nssdir $dstdir + + # Skip gtests when building. + sed '/^DIRS = /s/ cpputil gtests$//' $nssdir/manifest.mn > $dstdir/manifest.mn-t && mv $dstdir/manifest.mn-t $dstdir/manifest.mn + + # Remove subdirectories that we don't want. + rm -rf $dstdir/cmd + rm -rf $dstdir/tests + rm -rf $dstdir/lib + rm -rf $dstdir/pkg + rm -rf $dstdir/automation + rm -rf $dstdir/gtests + rm -rf $dstdir/cpputil + rm -rf $dstdir/doc + + # Start with an empty lib directory and copy only what we need. + mkdir $dstdir/lib + copy_top $nssdir/lib $dstdir/lib + cp -R $nssdir/lib/dbm $dstdir/lib/dbm + cp -R $nssdir/lib/freebl $dstdir/lib/freebl + cp -R $nssdir/lib/softoken $dstdir/lib/softoken + cp -R $nssdir/lib/sqlite $dstdir/lib/sqlite + + mkdir $dstdir/cmd + copy_top $nssdir/cmd $dstdir/cmd + cp -R $nssdir/cmd/bltest $dstdir/cmd/bltest + cp -R $nssdir/cmd/ecperf $dstdir/cmd/ecperf + cp -R $nssdir/cmd/fbectest $dstdir/cmd/fbectest + cp -R $nssdir/cmd/fipstest $dstdir/cmd/fipstest + cp -R $nssdir/cmd/lib $dstdir/cmd/lib + cp -R $nssdir/cmd/lowhashtest $dstdir/cmd/lowhashtest + cp -R $nssdir/cmd/shlibsign $dstdir/cmd/shlibsign + + mkdir $dstdir/tests + copy_top $nssdir/tests $dstdir/tests + + cp -R $nssdir/tests/cipher $dstdir/tests/cipher + cp -R $nssdir/tests/common $dstdir/tests/common + cp -R $nssdir/tests/ec $dstdir/tests/ec + cp -R $nssdir/tests/lowhash $dstdir/tests/lowhash + + cp $nssdir/lib/util/verref.h $dstdir/lib/freebl + cp $nssdir/lib/util/verref.h $dstdir/lib/softoken + cp $nssdir/lib/util/verref.h $dstdir/lib/softoken/legacydb +} + +split_nss() { + nssdir="$1" + dstdir="$2" + + # Prepare a source tree only containing files to build nss: + # + # nss/dbm full directory + # nss/coreconf full directory + # nss top files only + # nss/lib top files only + # nss/lib/freebl full directory + # nss/lib/softoken full directory + # nss/lib/softoken/dbm full directory + + # Copy everything. + cp -R $nssdir $dstdir + + # Remove subdirectories that we don't want. + rm -rf $dstdir/lib/freebl + rm -rf $dstdir/lib/softoken + rm -rf $dstdir/lib/util + rm -rf $dstdir/cmd/bltest + rm -rf $dstdir/cmd/fipstest + rm -rf $dstdir/cmd/rsaperf_low + + # Copy these headers until the upstream bug is accepted + # Upstream https://bugzilla.mozilla.org/show_bug.cgi?id=820207 + cp $nssdir/lib/softoken/lowkeyi.h $dstdir/cmd/rsaperf + cp $nssdir/lib/softoken/lowkeyti.h $dstdir/cmd/rsaperf + + # Copy verref.h which will be needed later during the build phase. + cp $nssdir/lib/util/verref.h $dstdir/lib/ckfw/builtins/verref.h + cp $nssdir/lib/util/verref.h $dstdir/lib/nss/verref.h + cp $nssdir/lib/util/verref.h $dstdir/lib/smime/verref.h + cp $nssdir/lib/util/verref.h $dstdir/lib/ssl/verref.h + cp $nssdir/lib/util/templates.c $dstdir/lib/nss/templates.c + + # FIXME: Skip util_gtest because it links with libnssutil.a. Note + # that we can't use libnssutil3.so instead, because util_gtest + # depends on internal symbols not exported from the shared library. + sed '/ util_gtest \\/d' $dstdir/gtests/manifest.mn > $dstdir/gtests/manifest.mn-t && mv $dstdir/gtests/manifest.mn-t $dstdir/gtests/manifest.mn +} diff --git a/security/nss/automation/taskcluster/scripts/tools.sh b/security/nss/automation/taskcluster/scripts/tools.sh index dacfdeb28..46d567e3a 100644 --- a/security/nss/automation/taskcluster/scripts/tools.sh +++ b/security/nss/automation/taskcluster/scripts/tools.sh @@ -2,11 +2,21 @@ set -v -e -x +if [[ $(id -u) -eq 0 ]]; then + # Drop privileges by re-running this script. + # Note: this mangles arguments, better to avoid running scripts as root. + exec su worker -c "$0 $*" +fi + # Usage: hg_clone repo dir [revision=@] hg_clone() { repo=$1 dir=$2 rev=${3:-@} + if [ -d "$dir" ]; then + hg pull -R "$dir" -ur "$rev" "$repo" && return + rm -rf "$dir" + fi for i in 0 2 5; do sleep $i hg clone -r "$rev" "$repo" "$dir" && return -- cgit v1.2.3