Update NSS to 3.32.1-RTM

14 files changed, 311 insertions, 116 deletions

diff --git a/security/nss/automation/taskcluster/scripts/build.sh b/security/nss/automation/taskcluster/scripts/build.sh
index 69968b138..649fdaa1b 100755
--- a/security/nss/automation/taskcluster/scripts/build.sh
+++ b/security/nss/automation/taskcluster/scripts/build.sh
@@ -1,14 +1,17 @@
 #!/usr/bin/env bash
 
-source $(dirname $0)/tools.sh
+source $(dirname "$0")/tools.sh
 
-if [[ $(id -u) -eq 0 ]]; then
-    # Drop privileges by re-running this script.
-    exec su worker $0
+if [ -n "$NSS_BUILD_MODULAR" ]; then
+    $(dirname "$0")/build_nspr.sh || exit $?
+    $(dirname "$0")/build_util.sh || exit $?
+    $(dirname "$0")/build_softoken.sh || exit $?
+    $(dirname "$0")/build_nss.sh || exit $?
+    exit
 fi
 
 # Clone NSPR if needed.
-hg_clone https://hg.mozilla.org/projects/nspr nspr default
+hg_clone https://hg.mozilla.org/projects/nspr ./nspr default
 
 # Build.
 make -C nss nss_build_all
diff --git a/security/nss/automation/taskcluster/scripts/build_gyp.sh b/security/nss/automation/taskcluster/scripts/build_gyp.sh
index 590e634a3..7190bd5c4 100755
--- a/security/nss/automation/taskcluster/scripts/build_gyp.sh
+++ b/security/nss/automation/taskcluster/scripts/build_gyp.sh
@@ -1,17 +1,12 @@
 #!/usr/bin/env bash
 
-source $(dirname $0)/tools.sh
-
-if [[ $(id -u) -eq 0 ]]; then
-    # Drop privileges by re-running this script.
-    exec su worker -c "$0 $*"
-fi
+source $(dirname "$0")/tools.sh
 
 # Clone NSPR if needed.
-hg_clone https://hg.mozilla.org/projects/nspr nspr default
+hg_clone https://hg.mozilla.org/projects/nspr ./nspr default
 
 # Build.
-nss/build.sh ${*--g -v}
+nss/build.sh -g -v "$@"
 
 # Package.
 mkdir artifacts
diff --git a/security/nss/automation/taskcluster/scripts/build_nspr.sh b/security/nss/automation/taskcluster/scripts/build_nspr.sh
new file mode 100644
index 000000000..4d1903460
--- /dev/null
+++ b/security/nss/automation/taskcluster/scripts/build_nspr.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -v -e -x
+
+source $(dirname $0)/tools.sh
+
+# Clone NSPR if needed.
+hg_clone https://hg.mozilla.org/projects/nspr nspr default
+
+# Build.
+rm -rf dist
+make -C nss build_nspr
+
+# Package.
+test -d artifacts || mkdir artifacts
+rm -rf dist-nspr
+mv dist dist-nspr
+tar cvfjh artifacts/dist-nspr.tar.bz2 dist-nspr
diff --git a/security/nss/automation/taskcluster/scripts/build_nss.sh b/security/nss/automation/taskcluster/scripts/build_nss.sh
new file mode 100644
index 000000000..b909bc30e
--- /dev/null
+++ b/security/nss/automation/taskcluster/scripts/build_nss.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+
+set -v -e -x
+
+source $(dirname $0)/tools.sh
+source $(dirname $0)/split.sh
+
+test -d dist-softoken || { echo "run build_softoken.sh first" 1>&2; exit 1; }
+
+rm -rf nss-nss
+split_nss nss nss-nss
+
+# Build.
+export NSS_BUILD_WITHOUT_SOFTOKEN=1
+export NSS_USE_SYSTEM_FREEBL=1
+
+platform=`make -s -C nss platform`
+
+export NSPR_LIB_DIR="$PWD/dist-nspr/$platform/lib"
+export NSSUTIL_LIB_DIR="$PWD/dist-util/$platform/lib"
+export FREEBL_LIB_DIR="$PWD/dist-softoken/$platform/lib"
+export SOFTOKEN_LIB_DIR="$PWD/dist-softoken/$platform/lib"
+export FREEBL_LIBS=-lfreebl
+
+export NSS_NO_PKCS11_BYPASS=1
+export FREEBL_NO_DEPEND=1
+
+export LIBRARY_PATH="$PWD/dist-nspr/$platform/lib:$PWD/dist-util/$platform/lib:$PWD/dist-softoken/$platform/lib"
+export LD_LIBRARY_PATH="$LIBRARY_PATH:$LD_LIBRARY_PATH"
+export INCLUDES="-I$PWD/dist-nspr/$platform/include -I$PWD/dist-util/public/nss -I$PWD/dist-softoken/public/nss"
+
+rm -rf dist
+make -C nss-nss nss_build_all
+
+# Package.
+test -d artifacts || mkdir artifacts
+rm -rf dist-nss
+mv dist dist-nss
+tar cvfjh artifacts/dist-nss.tar.bz2 dist-nss
diff --git a/security/nss/automation/taskcluster/scripts/build_softoken.sh b/security/nss/automation/taskcluster/scripts/build_softoken.sh
new file mode 100644
index 000000000..e5aaecccf
--- /dev/null
+++ b/security/nss/automation/taskcluster/scripts/build_softoken.sh
@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+set -v -e -x
+
+source $(dirname $0)/tools.sh
+source $(dirname $0)/split.sh
+
+test -d dist-util || { echo "run build_util.sh first" 1>&2; exit 1; }
+
+rm -rf nss-softoken
+split_softoken nss nss-softoken
+
+# Build.
+platform=`make -s -C nss platform`
+export LIBRARY_PATH="$PWD/dist-nspr/$platform/lib:$PWD/dist-util/$platform/lib"
+export LD_LIBRARY_PATH="$LIBRARY_PATH:$LD_LIBRARY_PATH"
+export INCLUDES="-I$PWD/dist-nspr/$platform/include -I$PWD/dist-util/public/nss"
+export NSS_BUILD_SOFTOKEN_ONLY=1
+
+rm -rf dist
+make -C nss-softoken nss_build_all
+
+mv dist/private/nss/blapi.h dist/public/nss
+mv dist/private/nss/alghmac.h dist/public/nss
+
+# Package.
+test -d artifacts || mkdir artifacts
+rm -rf dist-softoken
+mv dist dist-softoken
+tar cvfjh artifacts/dist-softoken.tar.bz2 dist-softoken
diff --git a/security/nss/automation/taskcluster/scripts/build_util.sh b/security/nss/automation/taskcluster/scripts/build_util.sh
new file mode 100644
index 000000000..0d2ecc5e8
--- /dev/null
+++ b/security/nss/automation/taskcluster/scripts/build_util.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -v -e -x
+
+source $(dirname $0)/tools.sh
+source $(dirname $0)/split.sh
+
+rm -rf nss-util
+split_util nss nss-util
+
+# Build.
+platform=`make -s -C nss platform`
+export LIBRARY_PATH="$PWD/dist-nspr/$platform/lib"
+export LD_LIBRARY_PATH="$LIBRARY_PATH:$LD_LIBRARY_PATH"
+export INCLUDES="-I$PWD/dist-nspr/$platform/include"
+export NSS_BUILD_UTIL_ONLY=1
+
+rm -rf dist
+make -C nss-util nss_build_all
+
+# Package.
+test -d artifacts || mkdir artifacts
+rm -rf dist-util
+mv dist dist-util
+tar cvfjh artifacts/dist-util.tar.bz2 dist-util
diff --git a/security/nss/automation/taskcluster/scripts/extend_task_graph.sh b/security/nss/automation/taskcluster/scripts/extend_task_graph.sh
index 5a3fb8d98..ade84cd60 100755
--- a/security/nss/automation/taskcluster/scripts/extend_task_graph.sh
+++ b/security/nss/automation/taskcluster/scripts/extend_task_graph.sh
@@ -1,11 +1,6 @@
 #!/usr/bin/env bash
 
-set -v -e -x
-
-if [ $(id -u) = 0 ]; then
-    # Drop privileges by re-running this script.
-    exec su worker $0
-fi
+source $(dirname "$0")/tools.sh
 
 mkdir -p /home/worker/artifacts
 
diff --git a/security/nss/automation/taskcluster/scripts/fuzz.sh b/security/nss/automation/taskcluster/scripts/fuzz.sh
index 5f8dd7bff..75851ff5b 100755
--- a/security/nss/automation/taskcluster/scripts/fuzz.sh
+++ b/security/nss/automation/taskcluster/scripts/fuzz.sh
@@ -1,20 +1,32 @@
 #!/usr/bin/env bash
 
-source $(dirname $0)/tools.sh
+source $(dirname "$0")/tools.sh
 
-if [ $(id -u) = 0 ]; then
-    # Drop privileges by re-running this script.
-    exec su worker -c "$0 $*"
-fi
+type="$1"
+shift
 
 # Fetch artifact if needed.
 fetch_dist
 
 # Clone corpus.
-./nss/fuzz/clone_corpus.sh
+./nss/fuzz/config/clone_corpus.sh
+
+# Ensure we have a corpus.
+if [ ! -d "nss/fuzz/corpus/$type" ]; then
+  mkdir -p nss/fuzz/corpus/$type
+
+  set +x
+
+  # Create a corpus out of what we have.
+  for f in $(find nss/fuzz/corpus -type f); do
+    cp $f "nss/fuzz/corpus/$type"
+  done
+
+  set -x
+fi
 
 # Fetch objdir name.
 objdir=$(cat dist/latest)
 
 # Run nssfuzz.
-LD_LIBRARY_PATH=$LD_LIBRARY_PATH:dist/$objdir/lib dist/$objdir/bin/nssfuzz $*
+dist/$objdir/bin/nssfuzz-"$type" "$@"
diff --git a/security/nss/automation/taskcluster/scripts/gen_certs.sh b/security/nss/automation/taskcluster/scripts/gen_certs.sh
index aee100147..b8d4f60ba 100755
--- a/security/nss/automation/taskcluster/scripts/gen_certs.sh
+++ b/security/nss/automation/taskcluster/scripts/gen_certs.sh
@@ -1,16 +1,6 @@
 #!/usr/bin/env bash
 
-set -v -e -x
-
-source $(dirname $0)/tools.sh
-
-if [ $(id -u) = 0 ]; then
-    # Stupid Docker.
-    echo "127.0.0.1 localhost.localdomain" >> /etc/hosts
-
-    # Drop privileges by re-running this script.
-    exec su worker $0
-fi
+source $(dirname "$0")/tools.sh
 
 # Fetch artifact if needed.
 fetch_dist
diff --git a/security/nss/automation/taskcluster/scripts/run_clang_format.sh b/security/nss/automation/taskcluster/scripts/run_clang_format.sh
deleted file mode 100755
index c4b60290f..000000000
--- a/security/nss/automation/taskcluster/scripts/run_clang_format.sh
+++ /dev/null
@@ -1,63 +0,0 @@
-#!/usr/bin/env bash
-
-set -v -e -x
-
-if [ $(id -u) -eq 0 ]; then
-    # Drop privileges by re-running this script.
-    exec su worker $0 "$@"
-fi
-
-# Apply clang-format on the provided folder and verify that this doesn't change any file.
-# If any file differs after formatting, the script eventually exits with 1.
-# Any differences between formatted and unformatted files is printed to stdout to give a hint what's wrong.
-
-# Includes a default set of directories.
-
-if [ $# -gt 0 ]; then
-    dirs=("$@")
-else
-    top=$(dirname $0)/../../..
-    dirs=( \
-         "$top/cmd" \
-         "$top/fuzz" \
-         "$top/lib/base" \
-         "$top/lib/certdb" \
-         "$top/lib/certhigh" \
-         "$top/lib/ckfw" \
-         "$top/lib/crmf" \
-         "$top/lib/cryptohi" \
-         "$top/lib/dbm" \
-         "$top/lib/dev" \
-         "$top/lib/freebl" \
-         "$top/lib/jar" \
-         "$top/lib/nss" \
-         "$top/lib/pk11wrap" \
-         "$top/lib/pkcs7" \
-         "$top/lib/pkcs12" \
-         "$top/lib/pki" \
-         "$top/lib/smime" \
-         "$top/lib/softoken" \
-         "$top/lib/ssl" \
-         "$top/lib/sysinit" \
-         "$top/lib/util" \
-         "$top/gtests/common" \
-         "$top/gtests/der_gtest" \
-         "$top/gtests/freebl_gtest" \
-         "$top/gtests/pk11_gtest" \
-         "$top/gtests/ssl_gtest" \
-         "$top/gtests/util_gtest" \
-    )
-fi
-
-for dir in "${dirs[@]}"; do
-    find "$dir" -type f \( -name '*.[ch]' -o -name '*.cc' \) -exec clang-format -i {} \+
-done
-
-TMPFILE=$(mktemp /tmp/$(basename $0).XXXXXX)
-trap 'rm $TMPFILE' exit
-if (cd $(dirname $0); hg root >/dev/null 2>&1); then
-    hg diff --git "$top" | tee $TMPFILE
-else
-    git -C "$top" diff | tee $TMPFILE
-fi
-[[ ! -s $TMPFILE ]]
diff --git a/security/nss/automation/taskcluster/scripts/run_scan_build.sh b/security/nss/automation/taskcluster/scripts/run_scan_build.sh
index 99f80ab5f..4024c226e 100755
--- a/security/nss/automation/taskcluster/scripts/run_scan_build.sh
+++ b/security/nss/automation/taskcluster/scripts/run_scan_build.sh
@@ -1,15 +1,10 @@
 #!/usr/bin/env bash
 
-source $(dirname $0)/tools.sh
-
-if [ $(id -u) = 0 ]; then
-    # Drop privileges by re-running this script.
-    exec su worker $0 $@
-fi
+source $(dirname "$0")/tools.sh
 
 # Clone NSPR if needed.
 if [ ! -d "nspr" ]; then
-    hg_clone https://hg.mozilla.org/projects/nspr nspr default
+    hg_clone https://hg.mozilla.org/projects/nspr ./nspr default
 fi
 
 # Build.
diff --git a/security/nss/automation/taskcluster/scripts/run_tests.sh b/security/nss/automation/taskcluster/scripts/run_tests.sh
index 4c87e7e32..b8e26761a 100755
--- a/security/nss/automation/taskcluster/scripts/run_tests.sh
+++ b/security/nss/automation/taskcluster/scripts/run_tests.sh
@@ -1,14 +1,6 @@
 #!/usr/bin/env bash
 
-source $(dirname $0)/tools.sh
-
-if [ $(id -u) = 0 ]; then
-    # Stupid Docker.
-    echo "127.0.0.1 localhost.localdomain" >> /etc/hosts
-
-    # Drop privileges by re-running this script.
-    exec su worker $0
-fi
+source $(dirname "$0")/tools.sh
 
 # Fetch artifact if needed.
 fetch_dist
diff --git a/security/nss/automation/taskcluster/scripts/split.sh b/security/nss/automation/taskcluster/scripts/split.sh
new file mode 100644
index 000000000..4d18385ec
--- /dev/null
+++ b/security/nss/automation/taskcluster/scripts/split.sh
@@ -0,0 +1,154 @@
+copy_top()
+{
+  srcdir_="$1"
+  dstdir_="$2"
+  files=`find "$srcdir_" -maxdepth 1 -mindepth 1 -type f`
+  for f in $files; do
+    cp -p "$f" "$dstdir_"
+  done
+}
+
+split_util() {
+  nssdir="$1"
+  dstdir="$2"
+
+  # Prepare a source tree only containing files to build nss-util:
+  #
+  #   nss/dbm                     full directory
+  #   nss/coreconf                full directory
+  #   nss                         top files only
+  #   nss/lib                     top files only
+  #   nss/lib/util                full directory
+
+  # Copy everything.
+  cp -R $nssdir $dstdir
+
+  # Skip gtests when building.
+  sed '/^DIRS = /s/ cpputil gtests$//' $nssdir/manifest.mn > $dstdir/manifest.mn-t && mv $dstdir/manifest.mn-t $dstdir/manifest.mn
+
+  # Remove subdirectories that we don't want.
+  rm -rf $dstdir/cmd
+  rm -rf $dstdir/tests
+  rm -rf $dstdir/lib
+  rm -rf $dstdir/automation
+  rm -rf $dstdir/gtests
+  rm -rf $dstdir/cpputil
+  rm -rf $dstdir/doc
+
+  # Start with an empty cmd lib directories to be filled selectively.
+  mkdir $dstdir/cmd
+  cp $nssdir/cmd/Makefile $dstdir/cmd
+  cp $nssdir/cmd/manifest.mn $dstdir/cmd
+  cp $nssdir/cmd/platlibs.mk $dstdir/cmd
+  cp $nssdir/cmd/platrules.mk $dstdir/cmd
+
+  # Copy some files at the top and the util subdirectory recursively.
+  mkdir $dstdir/lib
+  cp $nssdir/lib/Makefile $dstdir/lib
+  cp $nssdir/lib/manifest.mn $dstdir/lib
+  cp -R $nssdir/lib/util $dstdir/lib/util
+}
+
+split_softoken() {
+  nssdir="$1"
+  dstdir="$2"
+
+  # Prepare a source tree only containing files to build nss-softoken:
+  #
+  #   nss/dbm                     full directory
+  #   nss/coreconf                full directory
+  #   nss                         top files only
+  #   nss/lib                     top files only
+  #   nss/lib/freebl              full directory
+  #   nss/lib/softoken            full directory
+  #   nss/lib/softoken/dbm        full directory
+
+  # Copy everything.
+  cp -R $nssdir $dstdir
+
+  # Skip gtests when building.
+  sed '/^DIRS = /s/ cpputil gtests$//' $nssdir/manifest.mn > $dstdir/manifest.mn-t && mv $dstdir/manifest.mn-t $dstdir/manifest.mn
+
+  # Remove subdirectories that we don't want.
+  rm -rf $dstdir/cmd
+  rm -rf $dstdir/tests
+  rm -rf $dstdir/lib
+  rm -rf $dstdir/pkg
+  rm -rf $dstdir/automation
+  rm -rf $dstdir/gtests
+  rm -rf $dstdir/cpputil
+  rm -rf $dstdir/doc
+
+  # Start with an empty lib directory and copy only what we need.
+  mkdir $dstdir/lib
+  copy_top $nssdir/lib $dstdir/lib
+  cp -R $nssdir/lib/dbm $dstdir/lib/dbm
+  cp -R $nssdir/lib/freebl $dstdir/lib/freebl
+  cp -R $nssdir/lib/softoken $dstdir/lib/softoken
+  cp -R $nssdir/lib/sqlite $dstdir/lib/sqlite
+
+  mkdir $dstdir/cmd
+  copy_top $nssdir/cmd $dstdir/cmd
+  cp -R $nssdir/cmd/bltest $dstdir/cmd/bltest
+  cp -R $nssdir/cmd/ecperf $dstdir/cmd/ecperf
+  cp -R $nssdir/cmd/fbectest $dstdir/cmd/fbectest
+  cp -R $nssdir/cmd/fipstest $dstdir/cmd/fipstest
+  cp -R $nssdir/cmd/lib $dstdir/cmd/lib
+  cp -R $nssdir/cmd/lowhashtest $dstdir/cmd/lowhashtest
+  cp -R $nssdir/cmd/shlibsign $dstdir/cmd/shlibsign
+
+  mkdir $dstdir/tests
+  copy_top $nssdir/tests $dstdir/tests
+
+  cp -R $nssdir/tests/cipher $dstdir/tests/cipher
+  cp -R $nssdir/tests/common $dstdir/tests/common
+  cp -R $nssdir/tests/ec $dstdir/tests/ec
+  cp -R $nssdir/tests/lowhash $dstdir/tests/lowhash
+
+  cp $nssdir/lib/util/verref.h $dstdir/lib/freebl
+  cp $nssdir/lib/util/verref.h $dstdir/lib/softoken
+  cp $nssdir/lib/util/verref.h $dstdir/lib/softoken/legacydb
+}
+
+split_nss() {
+  nssdir="$1"
+  dstdir="$2"
+
+  # Prepare a source tree only containing files to build nss:
+  #
+  #   nss/dbm                     full directory
+  #   nss/coreconf                full directory
+  #   nss                         top files only
+  #   nss/lib                     top files only
+  #   nss/lib/freebl              full directory
+  #   nss/lib/softoken            full directory
+  #   nss/lib/softoken/dbm        full directory
+
+  # Copy everything.
+  cp -R $nssdir $dstdir
+
+  # Remove subdirectories that we don't want.
+  rm -rf $dstdir/lib/freebl
+  rm -rf $dstdir/lib/softoken
+  rm -rf $dstdir/lib/util
+  rm -rf $dstdir/cmd/bltest
+  rm -rf $dstdir/cmd/fipstest
+  rm -rf $dstdir/cmd/rsaperf_low
+
+  # Copy these headers until the upstream bug is accepted
+  # Upstream https://bugzilla.mozilla.org/show_bug.cgi?id=820207
+  cp $nssdir/lib/softoken/lowkeyi.h $dstdir/cmd/rsaperf
+  cp $nssdir/lib/softoken/lowkeyti.h $dstdir/cmd/rsaperf
+
+  # Copy verref.h which will be needed later during the build phase.
+  cp $nssdir/lib/util/verref.h $dstdir/lib/ckfw/builtins/verref.h
+  cp $nssdir/lib/util/verref.h $dstdir/lib/nss/verref.h
+  cp $nssdir/lib/util/verref.h $dstdir/lib/smime/verref.h
+  cp $nssdir/lib/util/verref.h $dstdir/lib/ssl/verref.h
+  cp $nssdir/lib/util/templates.c $dstdir/lib/nss/templates.c
+
+  # FIXME: Skip util_gtest because it links with libnssutil.a.  Note
+  # that we can't use libnssutil3.so instead, because util_gtest
+  # depends on internal symbols not exported from the shared library.
+  sed '/	util_gtest \\/d' $dstdir/gtests/manifest.mn > $dstdir/gtests/manifest.mn-t && mv $dstdir/gtests/manifest.mn-t $dstdir/gtests/manifest.mn
+}
diff --git a/security/nss/automation/taskcluster/scripts/tools.sh b/security/nss/automation/taskcluster/scripts/tools.sh
index dacfdeb28..46d567e3a 100644
--- a/security/nss/automation/taskcluster/scripts/tools.sh
+++ b/security/nss/automation/taskcluster/scripts/tools.sh
@@ -2,11 +2,21 @@
 
 set -v -e -x
 
+if [[ $(id -u) -eq 0 ]]; then
+    # Drop privileges by re-running this script.
+    # Note: this mangles arguments, better to avoid running scripts as root.
+    exec su worker -c "$0 $*"
+fi
+
 # Usage: hg_clone repo dir [revision=@]
 hg_clone() {
     repo=$1
     dir=$2
     rev=${3:-@}
+    if [ -d "$dir" ]; then
+        hg pull -R "$dir" -ur "$rev" "$repo" && return
+        rm -rf "$dir"
+    fi
     for i in 0 2 5; do
         sleep $i
         hg clone -r "$rev" "$repo" "$dir" && return