diff options
Diffstat (limited to 'security/nss/automation')
36 files changed, 737 insertions, 531 deletions
diff --git a/security/nss/automation/abi-check/expected-report-libnss3.so.txt b/security/nss/automation/abi-check/expected-report-libnss3.so.txt index e69de29bb..fa6e67412 100644 --- a/security/nss/automation/abi-check/expected-report-libnss3.so.txt +++ b/security/nss/automation/abi-check/expected-report-libnss3.so.txt @@ -0,0 +1,18 @@ + +1 function with some indirect sub-type change: + + [C]'function SECStatus CERT_AddOCSPAcceptableResponses(CERTOCSPRequest*, SECOidTag, ...)' at ocsp.c:2203:1 has some indirect sub-type changes: + parameter 2 of type 'typedef SECOidTag' has sub-type changes: + underlying type 'enum __anonymous_enum__' at secoidt.h:34:1 changed: + type size hasn't changed + 4 enumerator insertions: + '__anonymous_enum__::SEC_OID_X509_ANY_EXT_KEY_USAGE' value '357' + '__anonymous_enum__::SEC_OID_EXT_KEY_USAGE_IPSEC_IKE' value '358' + '__anonymous_enum__::SEC_OID_IPSEC_IKE_END' value '359' + '__anonymous_enum__::SEC_OID_IPSEC_IKE_INTERMEDIATE' value '360' + + 1 enumerator change: + '__anonymous_enum__::SEC_OID_TOTAL' from value '357' to '361' at secoidt.h:34:1 + + + diff --git a/security/nss/automation/abi-check/expected-report-libnssutil3.so.txt b/security/nss/automation/abi-check/expected-report-libnssutil3.so.txt index efc7d6d67..971365c68 100644 --- a/security/nss/automation/abi-check/expected-report-libnssutil3.so.txt +++ b/security/nss/automation/abi-check/expected-report-libnssutil3.so.txt @@ -1,4 +1,18 @@ -1 Added function: +1 function with some indirect sub-type change: + + [C]'function SECStatus NSS_GetAlgorithmPolicy(SECOidTag, PRUint32*)' at secoid.c:2217:1 has some indirect sub-type changes: + parameter 1 of type 'typedef SECOidTag' has sub-type changes: + underlying type 'enum __anonymous_enum__' at secoidt.h:34:1 changed: + type size hasn't changed + 4 enumerator insertions: + '__anonymous_enum__::SEC_OID_X509_ANY_EXT_KEY_USAGE' value '357' + '__anonymous_enum__::SEC_OID_EXT_KEY_USAGE_IPSEC_IKE' value '358' + '__anonymous_enum__::SEC_OID_IPSEC_IKE_END' value '359' + '__anonymous_enum__::SEC_OID_IPSEC_IKE_INTERMEDIATE' value '360' + + 1 enumerator change: + '__anonymous_enum__::SEC_OID_TOTAL' from value '357' to '361' at secoidt.h:34:1 + + - 'function SECStatus SECITEM_MakeItem(PLArenaPool*, SECItem*, unsigned char*, unsigned int)' {SECITEM_MakeItem@@NSSUTIL_3.38} diff --git a/security/nss/automation/abi-check/expected-report-libsmime3.so.txt b/security/nss/automation/abi-check/expected-report-libsmime3.so.txt index e69de29bb..f4870feec 100644 --- a/security/nss/automation/abi-check/expected-report-libsmime3.so.txt +++ b/security/nss/automation/abi-check/expected-report-libsmime3.so.txt @@ -0,0 +1,48 @@ + +1 function with some indirect sub-type change: + + [C]'function PK11SymKey* NSS_CMSContentInfo_GetBulkKey(NSSCMSContentInfo*)' at cmscinfo.c:363:1 has some indirect sub-type changes: + parameter 1 of type 'NSSCMSContentInfo*' has sub-type changes: + in pointed to type 'typedef NSSCMSContentInfo' at cmst.h:54:1: + underlying type 'struct NSSCMSContentInfoStr' at cmst.h:126:1 changed: + type size hasn't changed + 1 data member changes (2 filtered): + type of 'NSSCMSContent NSSCMSContentInfoStr::content' changed: + underlying type 'union NSSCMSContentUnion' at cmst.h:113:1 changed: + type size hasn't changed + 1 data member changes (3 filtered): + type of 'NSSCMSEncryptedData* NSSCMSContentUnion::encryptedData' changed: + in pointed to type 'typedef NSSCMSEncryptedData' at cmst.h:65:1: + underlying type 'struct NSSCMSEncryptedDataStr' at cmst.h:463:1 changed: + type size hasn't changed + 1 data member changes (1 filtered): + type of 'NSSCMSAttribute** NSSCMSEncryptedDataStr::unprotectedAttr' changed: + in pointed to type 'NSSCMSAttribute*': + in pointed to type 'typedef NSSCMSAttribute' at cmst.h:69:1: + underlying type 'struct NSSCMSAttributeStr' at cmst.h:482:1 changed: + type size hasn't changed + 1 data member change: + type of 'SECOidData* NSSCMSAttributeStr::typeTag' changed: + in pointed to type 'typedef SECOidData' at secoidt.h:16:1: + underlying type 'struct SECOidDataStr' at secoidt.h:513:1 changed: + type size hasn't changed + 1 data member change: + type of 'SECOidTag SECOidDataStr::offset' changed: + underlying type 'enum __anonymous_enum__' at secoidt.h:34:1 changed: + type size hasn't changed + 4 enumerator insertions: + '__anonymous_enum__::SEC_OID_X509_ANY_EXT_KEY_USAGE' value '357' + '__anonymous_enum__::SEC_OID_EXT_KEY_USAGE_IPSEC_IKE' value '358' + '__anonymous_enum__::SEC_OID_IPSEC_IKE_END' value '359' + '__anonymous_enum__::SEC_OID_IPSEC_IKE_INTERMEDIATE' value '360' + + 1 enumerator change: + '__anonymous_enum__::SEC_OID_TOTAL' from value '357' to '361' at secoidt.h:34:1 + + + + + + + + diff --git a/security/nss/automation/abi-check/previous-nss-release b/security/nss/automation/abi-check/previous-nss-release index c52061e7e..13a0361e3 100644 --- a/security/nss/automation/abi-check/previous-nss-release +++ b/security/nss/automation/abi-check/previous-nss-release @@ -1 +1 @@ -NSS_3_37_BRANCH +NSS_3_40_BRANCH diff --git a/security/nss/automation/clang-format/Dockerfile b/security/nss/automation/clang-format/Dockerfile index 163c9b8fa..e74dac09f 100644 --- a/security/nss/automation/clang-format/Dockerfile +++ b/security/nss/automation/clang-format/Dockerfile @@ -1,26 +1,35 @@ -FROM ubuntu:16.04 -MAINTAINER Franziskus Kiefer <franziskuskiefer@gmail.com> +# Minimal image with clang-format 3.9. +FROM ubuntu:18.04 +LABEL maintainer="Martin Thomson <martin.thomson@gmail.com>" -RUN useradd -d /home/worker -s /bin/bash -m worker -WORKDIR /home/worker +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + clang-format-3.9 \ + locales \ + mercurial \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get autoremove -y && apt-get clean -y -# Install dependencies. -ADD setup.sh /tmp/setup.sh -RUN bash /tmp/setup.sh +RUN update-alternatives --install /usr/bin/clang-format \ + clang-format $(which clang-format-3.9) 10 -# Change user. -USER worker - -# Env variables. -ENV HOME /home/worker ENV SHELL /bin/bash ENV USER worker -ENV LOGNAME worker +ENV LOGNAME $USER +ENV HOME /home/$USER ENV HOSTNAME taskcluster-worker ENV LANG en_US.UTF-8 -ENV LC_ALL en_US.UTF-8 +ENV LC_ALL $LANG ENV HOST localhost ENV DOMSUF localdomain -# Entrypoint. +RUN locale-gen $LANG \ + && DEBIAN_FRONTEND=noninteractive dpkg-reconfigure locales + +RUN useradd -d $HOME -s $SHELL -m $USER +WORKDIR $HOME +USER $USER + +# Entrypoint - which only works if /home/worker/nss is mounted. ENTRYPOINT ["/home/worker/nss/automation/clang-format/run_clang_format.sh"] diff --git a/security/nss/automation/clang-format/setup.sh b/security/nss/automation/clang-format/setup.sh deleted file mode 100644 index beac9e905..000000000 --- a/security/nss/automation/clang-format/setup.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash - -set -v -e -x - -# Update packages. -export DEBIAN_FRONTEND=noninteractive -apt-get -y update && apt-get -y upgrade - -# Install packages. -apt_packages=() -apt_packages+=('ca-certificates') -apt_packages+=('curl') -apt_packages+=('xz-utils') -apt_packages+=('mercurial') -apt_packages+=('git') -apt_packages+=('locales') -apt-get install -y --no-install-recommends ${apt_packages[@]} - -# Download clang. -curl -L https://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz -o clang.tar.xz -curl -L https://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz.sig -o clang.tar.xz.sig -# Verify the signature. -gpg --keyserver pool.sks-keyservers.net --recv-keys B6C8F98282B944E3B0D5C2530FC3042E345AD05D -gpg --verify clang.tar.xz.sig -# Install into /usr/local/. -tar xJvf *.tar.xz -C /usr/local --strip-components=1 - -# Cleanup. -function cleanup() { - rm -f clang.tar.xz clang.tar.xz.sig -} -trap cleanup ERR EXIT - -locale-gen en_US.UTF-8 -dpkg-reconfigure locales - -# Cleanup. -rm -rf ~/.ccache ~/.cache -apt-get autoremove -y -apt-get clean -apt-get autoclean - -# We're done. Remove this script. -rm $0 diff --git a/security/nss/automation/release/nspr-version.txt b/security/nss/automation/release/nspr-version.txt index 701680d2c..102def16d 100644 --- a/security/nss/automation/release/nspr-version.txt +++ b/security/nss/automation/release/nspr-version.txt @@ -1,4 +1,4 @@ -4.19 +4.20 # The first line of this file must contain the human readable NSPR # version number, which is the minimum required version of NSPR diff --git a/security/nss/automation/taskcluster/docker-aarch64/Dockerfile b/security/nss/automation/taskcluster/docker-aarch64/Dockerfile index 2d7ade357..aca173cd0 100644 --- a/security/nss/automation/taskcluster/docker-aarch64/Dockerfile +++ b/security/nss/automation/taskcluster/docker-aarch64/Dockerfile @@ -20,7 +20,6 @@ ENV HOME /home/worker ENV SHELL /bin/bash ENV USER worker ENV LOGNAME worker -ENV HOSTNAME taskcluster-worker ENV LANG en_US.UTF-8 ENV LC_ALL en_US.UTF-8 ENV HOST localhost diff --git a/security/nss/automation/taskcluster/docker-arm/Dockerfile b/security/nss/automation/taskcluster/docker-arm/Dockerfile index 9a7e50201..5b8cfca20 100644 --- a/security/nss/automation/taskcluster/docker-arm/Dockerfile +++ b/security/nss/automation/taskcluster/docker-arm/Dockerfile @@ -17,7 +17,6 @@ ENV HOME /home/worker ENV SHELL /bin/bash ENV USER worker ENV LOGNAME worker -ENV HOSTNAME taskcluster-worker ENV LANG en_US.UTF-8 ENV LC_ALL en_US.UTF-8 ENV HOST localhost diff --git a/security/nss/automation/taskcluster/docker-builds/Dockerfile b/security/nss/automation/taskcluster/docker-builds/Dockerfile new file mode 100644 index 000000000..9f0bb2034 --- /dev/null +++ b/security/nss/automation/taskcluster/docker-builds/Dockerfile @@ -0,0 +1,75 @@ +# Dockerfile for building extra builds. This includes more tools than the +# default image, so it's a fair bit bigger. Only use this for builds where +# the smaller docker image is missing something. These builds will run on +# the leaner configuration. +FROM ubuntu:18.04 +LABEL maintainer="Martin Thomson <martin.thomson@gmail.com>" + +RUN dpkg --add-architecture i386 +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + clang-4.0 \ + clang \ + cmake \ + curl \ + g++-4.8-multilib \ + g++-5-multilib \ + g++-6-multilib \ + g++-multilib \ + git \ + gyp \ + libelf-dev \ + libdw-dev \ + libssl-dev \ + libssl-dev:i386 \ + libxml2-utils \ + lib32z1-dev \ + linux-libc-dev:i386 \ + llvm-dev \ + locales \ + mercurial \ + ninja-build \ + pkg-config \ + valgrind \ + zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get autoremove -y && apt-get clean -y + +# Latest version of abigail-tools +RUN apt-get update \ + && apt-get install -y --no-install-recommends automake libtool libxml2-dev \ + && git clone git://sourceware.org/git/libabigail.git /tmp/libabigail \ + && cd /tmp/libabigail \ + && autoreconf -fi \ + && ./configure --prefix=/usr --disable-static --disable-apidoc --disable-manual \ + && make && make install \ + && rm -rf /tmp/libabigail \ + && apt-get remove -y automake libtool libxml2-dev \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get autoremove -y && apt-get clean -y + +ENV SHELL /bin/bash +ENV USER worker +ENV LOGNAME $USER +ENV HOME /home/$USER +ENV LANG en_US.UTF-8 +ENV LC_ALL $LANG +ENV HOST localhost +ENV DOMSUF localdomain + +RUN locale-gen $LANG \ + && DEBIAN_FRONTEND=noninteractive dpkg-reconfigure locales + +RUN useradd -d $HOME -s $SHELL -m $USER +WORKDIR $HOME + +# Add build and test scripts. +ADD bin $HOME/bin +RUN chmod +x $HOME/bin/* + +USER $USER + +# Set a default command for debugging. +CMD ["/bin/bash", "--login"] diff --git a/security/nss/automation/taskcluster/docker-clang-3.9/bin/checkout.sh b/security/nss/automation/taskcluster/docker-builds/bin/checkout.sh index 9167f6bda..9167f6bda 100644 --- a/security/nss/automation/taskcluster/docker-clang-3.9/bin/checkout.sh +++ b/security/nss/automation/taskcluster/docker-builds/bin/checkout.sh diff --git a/security/nss/automation/taskcluster/docker-clang-3.9/Dockerfile b/security/nss/automation/taskcluster/docker-clang-3.9/Dockerfile deleted file mode 100644 index 473ce64ba..000000000 --- a/security/nss/automation/taskcluster/docker-clang-3.9/Dockerfile +++ /dev/null @@ -1,30 +0,0 @@ -FROM ubuntu:16.04 -MAINTAINER Tim Taubert <ttaubert@mozilla.com> - -RUN useradd -d /home/worker -s /bin/bash -m worker -WORKDIR /home/worker - -# Add build and test scripts. -ADD bin /home/worker/bin -RUN chmod +x /home/worker/bin/* - -# Install dependencies. -ADD setup.sh /tmp/setup.sh -RUN bash /tmp/setup.sh - -# Change user. -USER worker - -# Env variables. -ENV HOME /home/worker -ENV SHELL /bin/bash -ENV USER worker -ENV LOGNAME worker -ENV HOSTNAME taskcluster-worker -ENV LANG en_US.UTF-8 -ENV LC_ALL en_US.UTF-8 -ENV HOST localhost -ENV DOMSUF localdomain - -# Set a default command for debugging. -CMD ["/bin/bash", "--login"] diff --git a/security/nss/automation/taskcluster/docker-clang-3.9/setup.sh b/security/nss/automation/taskcluster/docker-clang-3.9/setup.sh deleted file mode 100644 index 3076667a6..000000000 --- a/security/nss/automation/taskcluster/docker-clang-3.9/setup.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash - -set -v -e -x - -# Update packages. -export DEBIAN_FRONTEND=noninteractive -apt-get -y update && apt-get -y upgrade - -# Need this to add keys for PPAs below. -apt-get install -y --no-install-recommends apt-utils - -apt_packages=() -apt_packages+=('ca-certificates') -apt_packages+=('curl') -apt_packages+=('locales') -apt_packages+=('xz-utils') - -# Latest Mercurial. -apt_packages+=('mercurial') -apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 41BD8711B1F0EC2B0D85B91CF59CE3A8323293EE -echo "deb http://ppa.launchpad.net/mercurial-ppa/releases/ubuntu xenial main" > /etc/apt/sources.list.d/mercurial.list - -# Install packages. -apt-get -y update -apt-get install -y --no-install-recommends ${apt_packages[@]} - -# Download clang. -curl -LO https://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz -curl -LO https://releases.llvm.org/3.9.1/clang+llvm-3.9.1-x86_64-linux-gnu-ubuntu-16.04.tar.xz.sig -# Verify the signature. -gpg --keyserver pool.sks-keyservers.net --recv-keys B6C8F98282B944E3B0D5C2530FC3042E345AD05D -gpg --verify *.tar.xz.sig -# Install into /usr/local/. -tar xJvf *.tar.xz -C /usr/local --strip-components=1 -# Cleanup. -rm *.tar.xz* - -locale-gen en_US.UTF-8 -dpkg-reconfigure locales - -# Cleanup. -rm -rf ~/.ccache ~/.cache -apt-get autoremove -y -apt-get clean -apt-get autoclean -rm $0 diff --git a/security/nss/automation/taskcluster/docker-clang-format/Dockerfile b/security/nss/automation/taskcluster/docker-clang-format/Dockerfile new file mode 100644 index 000000000..c9f8b8b0e --- /dev/null +++ b/security/nss/automation/taskcluster/docker-clang-format/Dockerfile @@ -0,0 +1,38 @@ +# Minimal image with clang-format 3.9. +FROM ubuntu:18.04 +LABEL maintainer="Martin Thomson <martin.thomson@gmail.com>" + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + clang-format-3.9 \ + locales \ + mercurial \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get autoremove -y && apt-get clean -y + +RUN update-alternatives --install /usr/bin/clang-format \ + clang-format $(which clang-format-3.9) 10 + +ENV SHELL /bin/bash +ENV USER worker +ENV LOGNAME $USER +ENV HOME /home/$USER +ENV LANG en_US.UTF-8 +ENV LC_ALL $LANG +ENV HOST localhost +ENV DOMSUF localdomain + +RUN locale-gen $LANG \ + && DEBIAN_FRONTEND=noninteractive dpkg-reconfigure locales + +RUN useradd -d $HOME -s $SHELL -m $USER +WORKDIR $HOME + +ADD bin $HOME/bin +RUN chmod +x $HOME/bin/* + +USER $USER + +# Set a default command for debugging. +CMD ["/bin/bash", "--login"] diff --git a/security/nss/automation/taskcluster/docker-clang-format/bin/checkout.sh b/security/nss/automation/taskcluster/docker-clang-format/bin/checkout.sh new file mode 100644 index 000000000..9167f6bda --- /dev/null +++ b/security/nss/automation/taskcluster/docker-clang-format/bin/checkout.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -v -e -x + +if [ $(id -u) = 0 ]; then + # Drop privileges by re-running this script. + exec su worker $0 +fi + +# Default values for testing. +REVISION=${NSS_HEAD_REVISION:-default} +REPOSITORY=${NSS_HEAD_REPOSITORY:-https://hg.mozilla.org/projects/nss} + +# Clone NSS. +for i in 0 2 5; do + sleep $i + hg clone -r $REVISION $REPOSITORY nss && exit 0 + rm -rf nss +done +exit 1 diff --git a/security/nss/automation/taskcluster/docker-decision/Dockerfile b/security/nss/automation/taskcluster/docker-decision/Dockerfile index 473ce64ba..e0a31641e 100644 --- a/security/nss/automation/taskcluster/docker-decision/Dockerfile +++ b/security/nss/automation/taskcluster/docker-decision/Dockerfile @@ -1,30 +1,37 @@ -FROM ubuntu:16.04 -MAINTAINER Tim Taubert <ttaubert@mozilla.com> +# Minimal image for running the decision task. +FROM ubuntu:18.04 +LABEL maintainer="Martin Thomson <martin.thomson@gmail.com>" -RUN useradd -d /home/worker -s /bin/bash -m worker -WORKDIR /home/worker +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + locales \ + mercurial \ + nodejs \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get autoremove -y && apt-get clean -y -# Add build and test scripts. -ADD bin /home/worker/bin -RUN chmod +x /home/worker/bin/* - -# Install dependencies. -ADD setup.sh /tmp/setup.sh -RUN bash /tmp/setup.sh - -# Change user. -USER worker - -# Env variables. -ENV HOME /home/worker ENV SHELL /bin/bash ENV USER worker -ENV LOGNAME worker -ENV HOSTNAME taskcluster-worker +ENV LOGNAME $USER +ENV HOME /home/$USER ENV LANG en_US.UTF-8 -ENV LC_ALL en_US.UTF-8 +ENV LC_ALL $LANG ENV HOST localhost ENV DOMSUF localdomain +RUN locale-gen $LANG \ + && DEBIAN_FRONTEND=noninteractive dpkg-reconfigure locales + +RUN useradd -d $HOME -s $SHELL -m $USER +WORKDIR $HOME + +# Add build and test scripts. +ADD bin $HOME/bin +RUN chmod +x $HOME/bin/* + +USER $USER + # Set a default command for debugging. CMD ["/bin/bash", "--login"] diff --git a/security/nss/automation/taskcluster/docker-decision/setup.sh b/security/nss/automation/taskcluster/docker-decision/setup.sh deleted file mode 100644 index 51938529c..000000000 --- a/security/nss/automation/taskcluster/docker-decision/setup.sh +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env bash - -set -v -e -x - -# Update packages. -export DEBIAN_FRONTEND=noninteractive -apt-get -y update && apt-get -y upgrade - -# Need those to install newer packages below. -apt-get install -y --no-install-recommends apt-utils curl ca-certificates locales - -# Latest Mercurial. -apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 41BD8711B1F0EC2B0D85B91CF59CE3A8323293EE -echo "deb http://ppa.launchpad.net/mercurial-ppa/releases/ubuntu xenial main" > /etc/apt/sources.list.d/mercurial.list - -# Install packages. -apt-get -y update && apt-get install -y --no-install-recommends mercurial - -# Latest Node.JS. -curl -sL https://deb.nodesource.com/setup_6.x | bash - -apt-get install -y --no-install-recommends nodejs - -locale-gen en_US.UTF-8 -dpkg-reconfigure locales - -# Cleanup. -rm -rf ~/.ccache ~/.cache -apt-get autoremove -y -apt-get clean -apt-get autoclean -rm $0 diff --git a/security/nss/automation/taskcluster/docker-fuzz/Dockerfile b/security/nss/automation/taskcluster/docker-fuzz/Dockerfile index 254f166c8..24f939946 100644 --- a/security/nss/automation/taskcluster/docker-fuzz/Dockerfile +++ b/security/nss/automation/taskcluster/docker-fuzz/Dockerfile @@ -1,33 +1,59 @@ -FROM ubuntu:16.04 -MAINTAINER Tim Taubert <ttaubert@mozilla.com> +# Dockerfile for running fuzzing tests. +# +# Note that when running this, you need to add `--cap-add SYS_PTRACE` to the +# docker invocation or ASAN won't work. +# On taskcluster use `features: ["allowPtrace"]`. +# See https://github.com/google/sanitizers/issues/764#issuecomment-276700920 +FROM ubuntu:18.04 +LABEL maintainer="Martin Thomson <martin.thomson@gmail.com>" -RUN useradd -d /home/worker -s /bin/bash -m worker -WORKDIR /home/worker +RUN dpkg --add-architecture i386 +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + clang \ + clang-tools \ + curl \ + g++-multilib \ + git \ + gyp \ + libssl-dev \ + libssl-dev:i386 \ + libxml2-utils \ + lib32z1-dev \ + linux-libc-dev:i386 \ + llvm-dev \ + locales \ + mercurial \ + ninja-build \ + pkg-config \ + valgrind \ + zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get autoremove -y && apt-get clean -y -# Add build and test scripts. -ADD bin /home/worker/bin -RUN chmod +x /home/worker/bin/* - -# Install dependencies. -ADD setup.sh /tmp/setup.sh -RUN bash /tmp/setup.sh - -# Change user. -USER worker - -# Env variables. -ENV HOME /home/worker ENV SHELL /bin/bash ENV USER worker -ENV LOGNAME worker -ENV HOSTNAME taskcluster-worker +ENV LOGNAME $USER +ENV HOME /home/$USER ENV LANG en_US.UTF-8 -ENV LC_ALL en_US.UTF-8 +ENV LC_ALL $LANG ENV HOST localhost ENV DOMSUF localdomain -# LLVM 4.0 -ENV PATH "${PATH}:/home/worker/third_party/llvm-build/Release+Asserts/bin/" +RUN locale-gen $LANG \ + && DEBIAN_FRONTEND=noninteractive dpkg-reconfigure locales + +RUN useradd -d $HOME -s $SHELL -m $USER +WORKDIR $HOME + +# Add build and test scripts. +ADD bin $HOME/bin +RUN chmod +x $HOME/bin/* + +# Change user. +USER $USER # Set a default command for debugging. CMD ["/bin/bash", "--login"] diff --git a/security/nss/automation/taskcluster/docker-fuzz/setup.sh b/security/nss/automation/taskcluster/docker-fuzz/setup.sh deleted file mode 100644 index fcb72346e..000000000 --- a/security/nss/automation/taskcluster/docker-fuzz/setup.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env bash - -set -v -e -x - -# Update packages. -export DEBIAN_FRONTEND=noninteractive -apt-get -y update && apt-get -y upgrade - -# Need this to add keys for PPAs below. -apt-get install -y --no-install-recommends apt-utils - -apt_packages=() -apt_packages+=('build-essential') -apt_packages+=('ca-certificates') -apt_packages+=('curl') -apt_packages+=('git') -apt_packages+=('gyp') -apt_packages+=('libssl-dev') -apt_packages+=('libxml2-utils') -apt_packages+=('locales') -apt_packages+=('ninja-build') -apt_packages+=('pkg-config') -apt_packages+=('zlib1g-dev') - -# 32-bit builds -apt_packages+=('gcc-multilib') -apt_packages+=('g++-multilib') - -# Latest Mercurial. -apt_packages+=('mercurial') -apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 41BD8711B1F0EC2B0D85B91CF59CE3A8323293EE -echo "deb http://ppa.launchpad.net/mercurial-ppa/releases/ubuntu xenial main" > /etc/apt/sources.list.d/mercurial.list - -# Install packages. -apt-get -y update -apt-get install -y --no-install-recommends ${apt_packages[@]} - -# 32-bit builds -dpkg --add-architecture i386 -apt-get -y update -apt-get install -y --no-install-recommends libssl-dev:i386 - -# Install LLVM/clang-4.0. -mkdir clang-tmp -git clone -n --depth 1 https://chromium.googlesource.com/chromium/src/tools/clang clang-tmp/clang -git -C clang-tmp/clang checkout HEAD scripts/update.py -clang-tmp/clang/scripts/update.py -rm -fr clang-tmp - -locale-gen en_US.UTF-8 -dpkg-reconfigure locales - -# Cleanup. -rm -rf ~/.ccache ~/.cache -apt-get autoremove -y -apt-get clean -apt-get autoclean -rm $0 diff --git a/security/nss/automation/taskcluster/docker-fuzz32/Dockerfile b/security/nss/automation/taskcluster/docker-fuzz32/Dockerfile new file mode 100644 index 000000000..f5fd3cfd5 --- /dev/null +++ b/security/nss/automation/taskcluster/docker-fuzz32/Dockerfile @@ -0,0 +1,73 @@ +# Dockerfile for running fuzzing tests on linux32. +# +# This is a temporary workaround for bugs in clang that make it incompatible +# with Ubuntu 18.04 (see bug 1488148). This image can be removed once a new +# release of LLVM includes the necessary fixes. + +FROM ubuntu:16.04 +LABEL maintainer="Martin Thomson <martin.thomson@gmail.com>" + +RUN dpkg --add-architecture i386 +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + curl \ + g++-multilib \ + git \ + gyp \ + libssl-dev \ + libssl-dev:i386 \ + libxml2-utils \ + lib32z1-dev \ + linux-libc-dev:i386 \ + locales \ + mercurial \ + ninja-build \ + pkg-config \ + software-properties-common \ + valgrind \ + zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get autoremove -y && apt-get clean -y + +# Install clang and tools from the LLVM PPA. +RUN curl -sf https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - \ + && apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-6.0 main" \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + clang-6.0 \ + clang-tools-6.0 \ + llvm-6.0-dev \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get autoremove -y && apt-get clean -y + +# Alias all the clang commands. +RUN for i in $(dpkg -L clang-6.0 clang-tools-6.0 | grep '^/usr/bin/' | xargs -i basename {} -6.0); do \ + update-alternatives --install "/usr/bin/$i" "$i" "/usr/bin/${i}-6.0" 10; \ + done + +ENV SHELL /bin/bash +ENV USER worker +ENV LOGNAME $USER +ENV HOME /home/$USER +ENV LANG en_US.UTF-8 +ENV LC_ALL $LANG +ENV HOST localhost +ENV DOMSUF localdomain + +RUN locale-gen $LANG \ + && DEBIAN_FRONTEND=noninteractive dpkg-reconfigure locales + +RUN useradd -d $HOME -s $SHELL -m $USER +WORKDIR $HOME + +# Add build and test scripts. +ADD bin $HOME/bin +RUN chmod +x $HOME/bin/* + +# Change user. +USER $USER + +# Set a default command for debugging. +CMD ["/bin/bash", "--login"] diff --git a/security/nss/automation/taskcluster/docker-fuzz32/bin/checkout.sh b/security/nss/automation/taskcluster/docker-fuzz32/bin/checkout.sh new file mode 100644 index 000000000..9167f6bda --- /dev/null +++ b/security/nss/automation/taskcluster/docker-fuzz32/bin/checkout.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -v -e -x + +if [ $(id -u) = 0 ]; then + # Drop privileges by re-running this script. + exec su worker $0 +fi + +# Default values for testing. +REVISION=${NSS_HEAD_REVISION:-default} +REPOSITORY=${NSS_HEAD_REPOSITORY:-https://hg.mozilla.org/projects/nss} + +# Clone NSS. +for i in 0 2 5; do + sleep $i + hg clone -r $REVISION $REPOSITORY nss && exit 0 + rm -rf nss +done +exit 1 diff --git a/security/nss/automation/taskcluster/docker-gcc-4.4/Dockerfile b/security/nss/automation/taskcluster/docker-gcc-4.4/Dockerfile index 3330c007f..69538322a 100644 --- a/security/nss/automation/taskcluster/docker-gcc-4.4/Dockerfile +++ b/security/nss/automation/taskcluster/docker-gcc-4.4/Dockerfile @@ -1,30 +1,39 @@ FROM ubuntu:14.04 -MAINTAINER Tim Taubert <ttaubert@mozilla.com> +LABEL maintainer="Martin Thomson <martin.thomson@gmail.com>" -RUN useradd -d /home/worker -s /bin/bash -m worker -WORKDIR /home/worker +RUN dpkg --add-architecture i386 +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ca-certificates \ + g++-4.4 \ + gcc-4.4 \ + locales \ + make \ + mercurial \ + zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get autoremove -y && apt-get clean -y -# Add build and test scripts. -ADD bin /home/worker/bin -RUN chmod +x /home/worker/bin/* - -# Install dependencies. -ADD setup.sh /tmp/setup.sh -RUN bash /tmp/setup.sh - -# Change user. -USER worker - -# Env variables. -ENV HOME /home/worker ENV SHELL /bin/bash ENV USER worker -ENV LOGNAME worker -ENV HOSTNAME taskcluster-worker +ENV LOGNAME $USER +ENV HOME /home/$USER ENV LANG en_US.UTF-8 -ENV LC_ALL en_US.UTF-8 +ENV LC_ALL $LANG ENV HOST localhost ENV DOMSUF localdomain +RUN locale-gen $LANG \ + && DEBIAN_FRONTEND=noninteractive dpkg-reconfigure locales + +RUN useradd -d $HOME -s $SHELL -m $USER +WORKDIR $HOME + +# Add build and test scripts. +ADD bin $HOME/bin +RUN chmod +x $HOME/bin/* + +USER $USER + # Set a default command for debugging. CMD ["/bin/bash", "--login"] diff --git a/security/nss/automation/taskcluster/docker-gcc-4.4/setup.sh b/security/nss/automation/taskcluster/docker-gcc-4.4/setup.sh deleted file mode 100644 index f6325d966..000000000 --- a/security/nss/automation/taskcluster/docker-gcc-4.4/setup.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - -set -v -e -x - -# Update packages. -export DEBIAN_FRONTEND=noninteractive -apt-get -y update && apt-get -y upgrade - -apt_packages=() -apt_packages+=('ca-certificates') -apt_packages+=('g++-4.4') -apt_packages+=('gcc-4.4') -apt_packages+=('locales') -apt_packages+=('make') -apt_packages+=('mercurial') -apt_packages+=('zlib1g-dev') - -# Install packages. -apt-get -y update -apt-get install -y --no-install-recommends ${apt_packages[@]} - -locale-gen en_US.UTF-8 -dpkg-reconfigure locales - -# Cleanup. -rm -rf ~/.ccache ~/.cache -apt-get autoremove -y -apt-get clean -apt-get autoclean -rm $0 diff --git a/security/nss/automation/taskcluster/docker-interop/Dockerfile b/security/nss/automation/taskcluster/docker-interop/Dockerfile new file mode 100644 index 000000000..fb4e15d93 --- /dev/null +++ b/security/nss/automation/taskcluster/docker-interop/Dockerfile @@ -0,0 +1,56 @@ +# Dockerfile for running interop tests. +# This includes Rust, golang, and nodejs. +FROM ubuntu:18.04 +LABEL maintainer="Martin Thomson <martin.thomson@gmail.com>" + +RUN dpkg --add-architecture i386 +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + clang \ + cmake \ + curl \ + g++-multilib \ + git \ + golang \ + gyp \ + libxml2-utils \ + lib32z1-dev \ + linux-libc-dev:i386 \ + llvm-dev \ + locales \ + mercurial \ + ninja-build \ + npm \ + pkg-config \ + zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get autoremove -y && apt-get clean -y + +ENV SHELL /bin/bash +ENV USER worker +ENV LOGNAME $USER +ENV HOME /home/$USER +ENV LANG en_US.UTF-8 +ENV LC_ALL $LANG +ENV HOST localhost +ENV DOMSUF localdomain + +RUN locale-gen $LANG \ + && DEBIAN_FRONTEND=noninteractive dpkg-reconfigure locales + +RUN useradd -d $HOME -s $SHELL -m $USER +WORKDIR $HOME + +# Add build and test scripts. +ADD bin $HOME/bin +RUN chmod +x $HOME/bin/* + +USER $USER + +# Install Rust stable as $USER. +RUN curl https://sh.rustup.rs -sSf | sh -s -- -y + +# Set a default command for debugging. +CMD ["/bin/bash", "--login"] diff --git a/security/nss/automation/taskcluster/docker-interop/bin/checkout.sh b/security/nss/automation/taskcluster/docker-interop/bin/checkout.sh new file mode 100644 index 000000000..9167f6bda --- /dev/null +++ b/security/nss/automation/taskcluster/docker-interop/bin/checkout.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +set -v -e -x + +if [ $(id -u) = 0 ]; then + # Drop privileges by re-running this script. + exec su worker $0 +fi + +# Default values for testing. +REVISION=${NSS_HEAD_REVISION:-default} +REPOSITORY=${NSS_HEAD_REPOSITORY:-https://hg.mozilla.org/projects/nss} + +# Clone NSS. +for i in 0 2 5; do + sleep $i + hg clone -r $REVISION $REPOSITORY nss && exit 0 + rm -rf nss +done +exit 1 diff --git a/security/nss/automation/taskcluster/docker/Dockerfile b/security/nss/automation/taskcluster/docker/Dockerfile index b3c2516ba..6df17c5e1 100644 --- a/security/nss/automation/taskcluster/docker/Dockerfile +++ b/security/nss/automation/taskcluster/docker/Dockerfile @@ -1,30 +1,49 @@ -FROM ubuntu:16.04 -MAINTAINER Tim Taubert <ttaubert@mozilla.com> +# Lean image for running the bulk of the NSS CI tests on taskcluster. +FROM ubuntu:18.04 +LABEL maintainer="Martin Thomson <martin.thomson@gmail.com>" -RUN useradd -d /home/worker -s /bin/bash -m worker -WORKDIR /home/worker +RUN dpkg --add-architecture i386 +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + clang \ + curl \ + g++-multilib \ + git \ + gyp \ + libxml2-utils \ + lib32z1-dev \ + linux-libc-dev:i386 \ + llvm-dev \ + locales \ + mercurial \ + ninja-build \ + pkg-config \ + zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get autoremove -y && apt-get clean -y -# Add build and test scripts. -ADD bin /home/worker/bin -RUN chmod +x /home/worker/bin/* - -# Install dependencies. -ADD setup.sh /tmp/setup.sh -RUN bash /tmp/setup.sh - -# Env variables. -ENV HOME /home/worker ENV SHELL /bin/bash ENV USER worker -ENV LOGNAME worker -ENV HOSTNAME taskcluster-worker +ENV LOGNAME $USER +ENV HOME /home/$USER ENV LANG en_US.UTF-8 -ENV LC_ALL en_US.UTF-8 +ENV LC_ALL $LANG ENV HOST localhost ENV DOMSUF localdomain -# Rust + Go -ENV PATH "${PATH}:/home/worker/.cargo/bin/:/usr/lib/go-1.6/bin" +RUN locale-gen $LANG \ + && DEBIAN_FRONTEND=noninteractive dpkg-reconfigure locales + +RUN useradd -d $HOME -s $SHELL -m $USER +WORKDIR $HOME + +# Add build and test scripts. +ADD bin $HOME/bin +RUN chmod +x $HOME/bin/* + +USER $USER # Set a default command for debugging. CMD ["/bin/bash", "--login"] diff --git a/security/nss/automation/taskcluster/docker/setup.sh b/security/nss/automation/taskcluster/docker/setup.sh deleted file mode 100644 index 7b90b2e69..000000000 --- a/security/nss/automation/taskcluster/docker/setup.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/env bash - -set -v -e -x - -# Update packages. -export DEBIAN_FRONTEND=noninteractive -apt-get -y update && apt-get -y upgrade - -# Need this to add keys for PPAs below. -apt-get install -y --no-install-recommends apt-utils - -apt_packages=() -apt_packages+=('build-essential') -apt_packages+=('ca-certificates') -apt_packages+=('clang-5.0') -apt_packages+=('curl') -apt_packages+=('npm') -apt_packages+=('git') -apt_packages+=('golang-1.6') -apt_packages+=('libxml2-utils') -apt_packages+=('locales') -apt_packages+=('ninja-build') -apt_packages+=('pkg-config') -apt_packages+=('zlib1g-dev') - -# 32-bit builds -apt_packages+=('lib32z1-dev') -apt_packages+=('gcc-multilib') -apt_packages+=('g++-multilib') - -# ct-verif and sanitizers -apt_packages+=('valgrind') - -# Latest Mercurial. -apt_packages+=('mercurial') -apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 41BD8711B1F0EC2B0D85B91CF59CE3A8323293EE -echo "deb http://ppa.launchpad.net/mercurial-ppa/releases/ubuntu xenial main" > /etc/apt/sources.list.d/mercurial.list - -# gcc 4.8 and 6 -apt_packages+=('g++-6') -apt_packages+=('g++-4.8') -apt_packages+=('g++-6-multilib') -apt_packages+=('g++-4.8-multilib') -apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 60C317803A41BA51845E371A1E9377A2BA9EF27F -echo "deb http://ppa.launchpad.net/ubuntu-toolchain-r/test/ubuntu xenial main" > /etc/apt/sources.list.d/toolchain.list - -# Install packages. -apt-get -y update -apt-get install -y --no-install-recommends ${apt_packages[@]} - -# Latest version of abigail-tools -apt-get install -y libxml2-dev autoconf libelf-dev libdw-dev libtool -git clone git://sourceware.org/git/libabigail.git -cd ./libabigail -autoreconf -fi -./configure --prefix=/usr --disable-static --disable-apidoc --disable-manual -make -make install -cd .. -apt-get remove -y libxml2-dev autoconf libtool -rm -rf libabigail - -# Install latest Rust (stable). -su worker -c "curl https://sh.rustup.rs -sSf | sh -s -- -y" - -locale-gen en_US.UTF-8 -dpkg-reconfigure locales - -# Cleanup. -rm -rf ~/.ccache ~/.cache -apt-get autoremove -y -apt-get clean -apt-get autoclean -rm $0 diff --git a/security/nss/automation/taskcluster/graph/src/extend.js b/security/nss/automation/taskcluster/graph/src/extend.js index 5305325c5..1302602bc 100644 --- a/security/nss/automation/taskcluster/graph/src/extend.js +++ b/security/nss/automation/taskcluster/graph/src/extend.js @@ -10,9 +10,19 @@ const LINUX_IMAGE = { path: "automation/taskcluster/docker" }; -const LINUX_CLANG39_IMAGE = { - name: "linux-clang-3.9", - path: "automation/taskcluster/docker-clang-3.9" +const LINUX_BUILDS_IMAGE = { + name: "linux-builds", + path: "automation/taskcluster/docker-builds" +}; + +const LINUX_INTEROP_IMAGE = { + name: "linux-interop", + path: "automation/taskcluster/docker-interop" +}; + +const CLANG_FORMAT_IMAGE = { + name: "clang-format", + path: "automation/taskcluster/docker-clang-format" }; const LINUX_GCC44_IMAGE = { @@ -25,6 +35,12 @@ const FUZZ_IMAGE = { path: "automation/taskcluster/docker-fuzz" }; +// Bug 1488148 - temporary image for fuzzing 32-bit builds. +const FUZZ_IMAGE_32 = { + name: "fuzz32", + path: "automation/taskcluster/docker-fuzz32" +}; + const HACL_GEN_IMAGE = { name: "hacl", path: "automation/taskcluster/docker-hacl" @@ -59,7 +75,7 @@ queue.filter(task => { } } - if (task.tests == "bogo" || task.tests == "interop") { + if (task.tests == "bogo" || task.tests == "interop" || task.tests == "tlsfuzzer") { // No windows if (task.platform == "windows2012-64" || task.platform == "windows2012-32") { @@ -89,7 +105,9 @@ queue.filter(task => { if (task.group == "Test") { // Don't run test builds on old make platforms, and not for fips gyp. - if (task.collection == "make" || task.collection == "fips") { + // Disable on aarch64, see bug 1488331. + if (task.collection == "make" || task.collection == "fips" + || task.platform == "aarch64") { return false; } } @@ -134,13 +152,13 @@ export default async function main() { await scheduleLinux("Linux 32 (opt)", { platform: "linux32", image: LINUX_IMAGE - }, "-m32 --opt"); + }, "-t ia32 --opt"); await scheduleLinux("Linux 32 (debug)", { platform: "linux32", collection: "debug", image: LINUX_IMAGE - }, "-m32"); + }, "-t ia32"); await scheduleLinux("Linux 64 (opt)", { platform: "linux64", @@ -193,8 +211,8 @@ export default async function main() { UBSAN_OPTIONS: "print_stacktrace=1", NSS_DISABLE_ARENA_FREE_LIST: "1", NSS_DISABLE_UNLOAD: "1", - CC: "clang-5.0", - CCC: "clang++-5.0", + CC: "clang", + CCC: "clang++", }, platform: "linux64", collection: "asan", @@ -230,12 +248,12 @@ export default async function main() { await scheduleWindows("Windows 2012 32 (opt)", { platform: "windows2012-32", - }, "build_gyp.sh --opt -m32"); + }, "build_gyp.sh --opt -t ia32"); await scheduleWindows("Windows 2012 32 (debug)", { platform: "windows2012-32", collection: "debug" - }, "build_gyp.sh -m32"); + }, "build_gyp.sh -t ia32"); await scheduleFuzzing(); await scheduleFuzzing32(); @@ -251,29 +269,29 @@ export default async function main() { }; await scheduleLinux("Linux AArch64 (debug)", - merge({ + merge(aarch64_base, { command: [ "/bin/bash", "-c", "bin/checkout.sh && nss/automation/taskcluster/scripts/build_gyp.sh" ], collection: "debug", - }, aarch64_base) + }) ); await scheduleLinux("Linux AArch64 (opt)", - merge({ + merge(aarch64_base, { command: [ "/bin/bash", "-c", "bin/checkout.sh && nss/automation/taskcluster/scripts/build_gyp.sh --opt" ], collection: "opt", - }, aarch64_base) + }) ); await scheduleLinux("Linux AArch64 (debug, make)", - merge({ + merge(aarch64_base, { env: {USE_64: "1"}, command: [ "/bin/bash", @@ -281,7 +299,7 @@ export default async function main() { "bin/checkout.sh && nss/automation/taskcluster/scripts/build.sh" ], collection: "make", - }, aarch64_base) + }) ); await scheduleMac("Mac (opt)", {collection: "opt"}, "--opt"); @@ -303,7 +321,7 @@ async function scheduleMac(name, base, args = "") { }); // Build base definition. - let build_base = merge({ + let build_base = merge(mac_base, { command: [ MAC_CHECKOUT_CMD, ["bash", "-c", @@ -320,7 +338,7 @@ async function scheduleMac(name, base, args = "") { }], kind: "build", symbol: "B" - }, mac_base); + }); // The task that builds NSPR+NSS. let task_build = queue.scheduleTask(merge(build_base, {name})); @@ -351,14 +369,18 @@ async function scheduleMac(name, base, args = "") { /*****************************************************************************/ -async function scheduleLinux(name, base, args = "") { - // Build base definition. - let build_base = merge({ +async function scheduleLinux(name, overrides, args = "") { + // Construct a base definition. This takes |overrides| second because + // callers expect to be able to overwrite the |command| key. + let base = merge({ command: [ "/bin/bash", "-c", "bin/checkout.sh && nss/automation/taskcluster/scripts/build_gyp.sh " + args ], + }, overrides); + // The base for building. + let build_base = merge(base, { artifacts: { public: { expires: 24 * 7, @@ -367,8 +389,8 @@ async function scheduleLinux(name, base, args = "") { } }, kind: "build", - symbol: "B" - }, base); + symbol: "B", + }); // The task that builds NSPR+NSS. let task_build = queue.scheduleTask(merge(build_base, {name})); @@ -434,14 +456,17 @@ async function scheduleLinux(name, base, args = "") { })); // Extra builds. - let extra_base = merge({group: "Builds"}, build_base); + let extra_base = merge(build_base, { + group: "Builds", + image: LINUX_BUILDS_IMAGE, + }); queue.scheduleTask(merge(extra_base, { - name: `${name} w/ clang-5.0`, + name: `${name} w/ clang-4`, env: { - CC: "clang-5.0", - CCC: "clang++-5.0", + CC: "clang-4.0", + CCC: "clang++-4.0", }, - symbol: "clang-5.0" + symbol: "clang-4" })); queue.scheduleTask(merge(extra_base, { @@ -474,16 +499,26 @@ async function scheduleLinux(name, base, args = "") { })); queue.scheduleTask(merge(extra_base, { - name: `${name} w/ gcc-6.1`, + name: `${name} w/ gcc-5`, + env: { + CC: "gcc-5", + CCC: "g++-5" + }, + symbol: "gcc-5" + })); + + queue.scheduleTask(merge(extra_base, { + name: `${name} w/ gcc-6`, env: { CC: "gcc-6", CCC: "g++-6" }, - symbol: "gcc-6.1" + symbol: "gcc-6" })); queue.scheduleTask(merge(extra_base, { name: `${name} w/ modular builds`, + image: LINUX_IMAGE, env: {NSS_BUILD_MODULAR: "1"}, command: [ "/bin/bash", @@ -493,7 +528,7 @@ async function scheduleLinux(name, base, args = "") { symbol: "modular" })); - await scheduleTestBuilds(merge(base, {group: "Test"}), args); + await scheduleTestBuilds(name + " Test", merge(base, {group: "Test"}), args); return queue.submit(); } @@ -534,7 +569,7 @@ async function scheduleFuzzing() { }; // Build base definition. - let build_base = merge({ + let build_base = merge(base, { command: [ "/bin/bash", "-c", @@ -550,7 +585,7 @@ async function scheduleFuzzing() { }, kind: "build", symbol: "B" - }, base); + }); // The task that builds NSPR+NSS. let task_build = queue.scheduleTask(merge(build_base, { @@ -635,16 +670,16 @@ async function scheduleFuzzing32() { features: ["allowPtrace"], platform: "linux32", collection: "fuzz", - image: FUZZ_IMAGE + image: FUZZ_IMAGE_32 }; // Build base definition. - let build_base = merge({ + let build_base = merge(base, { command: [ "/bin/bash", "-c", "bin/checkout.sh && " + - "nss/automation/taskcluster/scripts/build_gyp.sh -g -v --fuzz -m32" + "nss/automation/taskcluster/scripts/build_gyp.sh -g -v --fuzz -t ia32" ], artifacts: { public: { @@ -655,7 +690,7 @@ async function scheduleFuzzing32() { }, kind: "build", symbol: "B" - }, base); + }); // The task that builds NSPR+NSS. let task_build = queue.scheduleTask(merge(build_base, { @@ -671,7 +706,7 @@ async function scheduleFuzzing32() { "/bin/bash", "-c", "bin/checkout.sh && " + - "nss/automation/taskcluster/scripts/build_gyp.sh -g -v --fuzz=tls -m32" + "nss/automation/taskcluster/scripts/build_gyp.sh -g -v --fuzz=tls -t ia32" ], })); @@ -728,9 +763,9 @@ async function scheduleFuzzing32() { /*****************************************************************************/ -async function scheduleTestBuilds(base, args = "") { +async function scheduleTestBuilds(name, base, args = "") { // Build base definition. - let build = merge({ + let build = merge(base, { command: [ "/bin/bash", "-c", @@ -746,8 +781,15 @@ async function scheduleTestBuilds(base, args = "") { }, kind: "build", symbol: "B", - name: "Linux 64 (debug, test)" - }, base); + name: `${name} build`, + }); + + // On linux we have a specialized build image for building. + if (build.platform === "linux32" || build.platform === "linux64") { + build = merge(build, { + image: LINUX_BUILDS_IMAGE, + }); + } // The task that builds NSPR+NSS. let task_build = queue.scheduleTask(build); @@ -755,7 +797,7 @@ async function scheduleTestBuilds(base, args = "") { // Schedule tests. queue.scheduleTask(merge(base, { parent: task_build, - name: "mpi", + name: `${name} mpi tests`, command: [ "/bin/bash", "-c", @@ -773,7 +815,7 @@ async function scheduleTestBuilds(base, args = "") { "-c", "bin/checkout.sh && nss/automation/taskcluster/scripts/run_tests.sh" ], - name: "Gtests", + name: `${name} gtests`, symbol: "Gtest", tests: "gtests", cycle: "standard", @@ -790,12 +832,12 @@ async function scheduleWindows(name, base, build_script) { base = merge(base, { workerType: "nss-win2012r2", env: { - PATH: "c:\\mozilla-build\\python;c:\\mozilla-build\\msys\\local\\bin;" + - "c:\\mozilla-build\\7zip;c:\\mozilla-build\\info-zip;" + - "c:\\mozilla-build\\python\\Scripts;c:\\mozilla-build\\yasm;" + - "c:\\mozilla-build\\msys\\bin;c:\\Windows\\system32;" + - "c:\\mozilla-build\\upx391w;c:\\mozilla-build\\moztools-x64\\bin;" + - "c:\\mozilla-build\\wget", + PATH: "c:\\mozilla-build\\bin;c:\\mozilla-build\\python;" + + "c:\\mozilla-build\\msys\\local\\bin;c:\\mozilla-build\\7zip;" + + "c:\\mozilla-build\\info-zip;c:\\mozilla-build\\python\\Scripts;" + + "c:\\mozilla-build\\yasm;c:\\mozilla-build\\msys\\bin;" + + "c:\\Windows\\system32;c:\\mozilla-build\\upx391w;" + + "c:\\mozilla-build\\moztools-x64\\bin;c:\\mozilla-build\\wget", DOMSUF: "localdomain", HOST: "localhost", } @@ -881,7 +923,7 @@ async function scheduleWindows(name, base, build_script) { /*****************************************************************************/ function scheduleTests(task_build, task_cert, test_base) { - test_base = merge({kind: "test"}, test_base); + test_base = merge(test_base, {kind: "test"}); // Schedule tests that do NOT need certificates. let no_cert_base = merge(test_base, {parent: task_build}); @@ -889,10 +931,21 @@ function scheduleTests(task_build, task_cert, test_base) { name: "Gtests", symbol: "Gtest", tests: "ssl_gtests gtests", cycle: "standard" })); queue.scheduleTask(merge(no_cert_base, { - name: "Bogo tests", symbol: "Bogo", tests: "bogo", cycle: "standard" + name: "Bogo tests", + symbol: "Bogo", + tests: "bogo", + cycle: "standard", + image: LINUX_INTEROP_IMAGE, })); queue.scheduleTask(merge(no_cert_base, { - name: "Interop tests", symbol: "Interop", tests: "interop", cycle: "standard" + name: "Interop tests", + symbol: "Interop", + tests: "interop", + cycle: "standard", + image: LINUX_INTEROP_IMAGE, + })); + queue.scheduleTask(merge(no_cert_base, { + name: "tlsfuzzer tests", symbol: "tlsfuzzer", tests: "tlsfuzzer", cycle: "standard" })); queue.scheduleTask(merge(no_cert_base, { name: "Chains tests", symbol: "Chains", tests: "chains" @@ -928,6 +981,9 @@ function scheduleTests(task_build, task_cert, test_base) { queue.scheduleTask(merge(no_cert_base, { name: "SDR tests", symbol: "SDR", tests: "sdr" })); + queue.scheduleTask(merge(no_cert_base, { + name: "Policy tests", symbol: "Policy", tests: "policy" + })); // Schedule tests that need certificates. let cert_base = merge(test_base, {parent: task_cert}); @@ -971,11 +1027,11 @@ async function scheduleTools() { kind: "test" }; - //ABI check task + // ABI check task queue.scheduleTask(merge(base, { symbol: "abi", name: "abi", - image: LINUX_IMAGE, + image: LINUX_BUILDS_IMAGE, command: [ "/bin/bash", "-c", @@ -984,9 +1040,9 @@ async function scheduleTools() { })); queue.scheduleTask(merge(base, { - symbol: "clang-format-3.9", - name: "clang-format-3.9", - image: LINUX_CLANG39_IMAGE, + symbol: "clang-format", + name: "clang-format", + image: CLANG_FORMAT_IMAGE, command: [ "/bin/bash", "-c", @@ -1049,7 +1105,7 @@ async function scheduleTools() { command: [ "/bin/bash", "-c", - "bin/checkout.sh && nss/automation/taskcluster/scripts/build_gyp.sh --disable-tests --emit-llvm -m32" + "bin/checkout.sh && nss/automation/taskcluster/scripts/build_gyp.sh --disable-tests --emit-llvm -t ia32" ] })); diff --git a/security/nss/automation/taskcluster/graph/src/try_syntax.js b/security/nss/automation/taskcluster/graph/src/try_syntax.js index 214793bd5..f1772a658 100644 --- a/security/nss/automation/taskcluster/graph/src/try_syntax.js +++ b/security/nss/automation/taskcluster/graph/src/try_syntax.js @@ -37,7 +37,7 @@ function parseOptions(opts) { let aliases = {"gtests": "gtest"}; let allUnitTests = ["bogo", "crmf", "chains", "cipher", "db", "ec", "fips", "gtest", "interop", "lowhash", "merge", "sdr", "smime", "tools", - "ssl", "mpi", "scert", "spki"]; + "ssl", "mpi", "scert", "spki", "policy", "tlsfuzzer"]; let unittests = intersect(opts.unittests.split(/\s*,\s*/).map(t => { return aliases[t] || t; }), allUnitTests); diff --git a/security/nss/automation/taskcluster/scripts/build_image.sh b/security/nss/automation/taskcluster/scripts/build_image.sh index b422214e7..b8715dbe9 100644 --- a/security/nss/automation/taskcluster/scripts/build_image.sh +++ b/security/nss/automation/taskcluster/scripts/build_image.sh @@ -13,12 +13,12 @@ raise_error() { test -n "$PROJECT" || raise_error "Project must be provided." test -n "$HASH" || raise_error "Context Hash must be provided." -CONTEXT_PATH=/home/worker/nss/$CONTEXT_PATH +CONTEXT_PATH="/home/worker/nss/$CONTEXT_PATH" -test -d $CONTEXT_PATH || raise_error "Context Path $CONTEXT_PATH does not exist." +test -d "$CONTEXT_PATH" || raise_error "Context Path $CONTEXT_PATH does not exist." test -f "$CONTEXT_PATH/Dockerfile" || raise_error "Dockerfile must be present in $CONTEXT_PATH." -docker build -t $PROJECT:$HASH $CONTEXT_PATH +docker build -t "$PROJECT:$HASH" "$CONTEXT_PATH" mkdir /artifacts -docker save $PROJECT:$HASH > /artifacts/image.tar +docker save "$PROJECT:$HASH" > /artifacts/image.tar diff --git a/security/nss/automation/taskcluster/scripts/tools.sh b/security/nss/automation/taskcluster/scripts/tools.sh index 534cb32ce..63a6ee8de 100644 --- a/security/nss/automation/taskcluster/scripts/tools.sh +++ b/security/nss/automation/taskcluster/scripts/tools.sh @@ -2,12 +2,11 @@ set -v -e -x +# Assert that we're not running as root. if [[ $(id -u) -eq 0 ]]; then - # Stupid Docker. It works without sometimes... But not always. - echo "127.0.0.1 localhost.localdomain" >> /etc/hosts - - # Drop privileges by re-running this script. - # Note: this mangles arguments, better to avoid running scripts as root. + # This exec is still needed until aarch64 images are updated (Bug 1488325). + # Remove when images are updated. Until then, assert that things are good. + [[ $(uname -m) == aarch64 ]] exec su worker -c "$0 $*" fi diff --git a/security/nss/automation/taskcluster/windows/build.sh b/security/nss/automation/taskcluster/windows/build.sh index 46136153d..eebb41535 100644 --- a/security/nss/automation/taskcluster/windows/build.sh +++ b/security/nss/automation/taskcluster/windows/build.sh @@ -2,12 +2,12 @@ set -v -e -x -# Set up the toolchain. -if [ "$USE_64" = 1 ]; then - source $(dirname $0)/setup64.sh +if [[ "$USE_64" == 1 ]]; then + m=x64 else - source $(dirname $0)/setup32.sh + m=x86 fi +source "$(dirname "$0")/setup.sh" # Clone NSPR. hg_clone https://hg.mozilla.org/projects/nspr nspr default diff --git a/security/nss/automation/taskcluster/windows/build_gyp.sh b/security/nss/automation/taskcluster/windows/build_gyp.sh index cc829ca99..c0f38f948 100644 --- a/security/nss/automation/taskcluster/windows/build_gyp.sh +++ b/security/nss/automation/taskcluster/windows/build_gyp.sh @@ -2,33 +2,37 @@ set -v -e -x -# Set up the toolchain. -if [[ "$@" == *"-m32"* ]]; then - source $(dirname $0)/setup32.sh -else - source $(dirname $0)/setup64.sh -fi +# Parse for the -t option. +m=x64 +for i in "$@"; do + case "$i" in + -t|--target) m= ;; + --target=*) m="${i#*=}" ;; + *) [[ -z "$m" ]] && m="$i" ;; + esac +done +[[ "$m" == "ia32" ]] && m=x86 +source "$(dirname "$0")/setup.sh" # Install GYP. -cd gyp +pushd gyp python -m virtualenv test-env test-env/Scripts/python setup.py install test-env/Scripts/python -m pip install --upgrade pip test-env/Scripts/pip install --upgrade setuptools -cd .. - -export GYP_MSVS_OVERRIDE_PATH="${VSPATH}" -export GYP_MSVS_VERSION="2015" -export GYP="${PWD}/gyp/test-env/Scripts/gyp" - # Fool GYP. touch "${VSPATH}/VC/vcvarsall.bat" +export GYP_MSVS_OVERRIDE_PATH="${VSPATH}" +export GYP_MSVS_VERSION=2015 +popd + +export PATH="${PATH}:${PWD}/ninja/bin:${PWD}/gyp/test-env/Scripts" # Clone NSPR. hg_clone https://hg.mozilla.org/projects/nspr nspr default # Build with gyp. -GYP=${GYP} ./nss/build.sh -g -v "$@" +./nss/build.sh -g -v "$@" # Package. 7z a public/build/dist.7z dist diff --git a/security/nss/automation/taskcluster/windows/setup.sh b/security/nss/automation/taskcluster/windows/setup.sh index 36a040ba1..93c0cdbd5 100644 --- a/security/nss/automation/taskcluster/windows/setup.sh +++ b/security/nss/automation/taskcluster/windows/setup.sh @@ -2,13 +2,6 @@ set -v -e -x -export VSPATH="$(pwd)/vs2017_15.4.2" -export NINJA_PATH="$(pwd)/ninja/bin" - -export WINDOWSSDKDIR="${VSPATH}/SDK" -export VS90COMNTOOLS="${VSPATH}/VC" -export INCLUDE="${VSPATH}/VC/include:${VSPATH}/SDK/Include/10.0.15063.0/ucrt:${VSPATH}/SDK/Include/10.0.15063.0/shared:${VSPATH}/SDK/Include/10.0.15063.0/um" - # Usage: hg_clone repo dir [revision=@] hg_clone() { repo=$1 @@ -22,5 +15,42 @@ hg_clone() { exit 1 } -hg_clone https://hg.mozilla.org/build/tools tools default -tools/scripts/tooltool/tooltool_wrapper.sh $(dirname $0)/releng.manifest https://tooltool.mozilla-releng.net/ non-existant-file.sh /c/mozilla-build/python/python.exe /c/builds/tooltool.py --authentication-file /c/builds/relengapi.tok -c /c/builds/tooltool_cache +hg_clone https://hg.mozilla.org/build/tools tools b8d7c263dfc3 +tools/scripts/tooltool/tooltool_wrapper.sh \ + $(dirname $0)/releng.manifest https://tooltool.mozilla-releng.net/ \ + non-existant-file.sh /c/mozilla-build/python/python.exe \ + /c/builds/tooltool.py --authentication-file /c/builds/relengapi.tok \ + -c /c/builds/tooltool_cache + +# This needs $m to be set. +[[ -n "$m" ]] + +# Setup MSVC paths. +export VSPATH="${PWD}/vs2017_15.4.2" +UCRTVersion="10.0.15063.0" + +export WINDOWSSDKDIR="${VSPATH}/SDK" +export VS90COMNTOOLS="${VSPATH}/VC" +export WIN32_REDIST_DIR="${VSPATH}/VC/redist/${m}/Microsoft.VC141.CRT" +export WIN_UCRT_REDIST_DIR="${VSPATH}/SDK/Redist/ucrt/DLLs/${m}" + +if [ "$m" == "x86" ]; then + PATH="${PATH}:${VSPATH}/VC/bin/Hostx64/x86" + PATH="${PATH}:${VSPATH}/VC/bin/Hostx64/x64" +fi +PATH="${PATH}:${VSPATH}/VC/bin/Host${m}/${m}" +PATH="${PATH}:${WIN32_REDIST_DIR}" +PATH="${PATH}:${WIN_UCRT_REDIST_DIR}" +PATH="${PATH}:${VSPATH}/SDK/bin/${UCRTVersion}/x64" +export PATH + +LIB="${LIB}:${VSPATH}/VC/lib/${m}" +LIB="${LIB}:${VSPATH}/SDK/lib/${UCRTVersion}/ucrt/${m}" +LIB="${LIB}:${VSPATH}/SDK/lib/${UCRTVersion}/um/${m}" +export LIB + +INCLUDE="${INCLUDE}:${VSPATH}/VC/include" +INCLUDE="${INCLUDE}:${VSPATH}/SDK/Include/${UCRTVersion}/ucrt" +INCLUDE="${INCLUDE}:${VSPATH}/SDK/Include/${UCRTVersion}/shared" +INCLUDE="${INCLUDE}:${VSPATH}/SDK/Include/${UCRTVersion}/um" +export INCLUDE diff --git a/security/nss/automation/taskcluster/windows/setup32.sh b/security/nss/automation/taskcluster/windows/setup32.sh deleted file mode 100644 index 19bed284d..000000000 --- a/security/nss/automation/taskcluster/windows/setup32.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -set -v -e -x - -source $(dirname $0)/setup.sh - -export WIN32_REDIST_DIR="${VSPATH}/VC/redist/x86/Microsoft.VC141.CRT" -export WIN_UCRT_REDIST_DIR="${VSPATH}/SDK/Redist/ucrt/DLLs/x86" -export PATH="${NINJA_PATH}:${VSPATH}/VC/bin/Hostx64/x86:${VSPATH}/VC/bin/Hostx64/x64:${VSPATH}/VC/Hostx86/x86:${VSPATH}/SDK/bin/10.0.15063.0/x64:${VSPATH}/VC/redist/x86/Microsoft.VC141.CRT:${VSPATH}/SDK/Redist/ucrt/DLLs/x86:${PATH}" -export LIB="${VSPATH}/VC/lib/x86:${VSPATH}/SDK/lib/10.0.15063.0/ucrt/x86:${VSPATH}/SDK/lib/10.0.15063.0/um/x86" diff --git a/security/nss/automation/taskcluster/windows/setup64.sh b/security/nss/automation/taskcluster/windows/setup64.sh deleted file mode 100644 index d16cb0ec9..000000000 --- a/security/nss/automation/taskcluster/windows/setup64.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/usr/bin/env bash - -set -v -e -x - -source $(dirname $0)/setup.sh - -export WIN32_REDIST_DIR="${VSPATH}/VC/redist/x64/Microsoft.VC141.CRT" -export WIN_UCRT_REDIST_DIR="${VSPATH}/SDK/Redist/ucrt/DLLs/x64" -export PATH="${NINJA_PATH}:${VSPATH}/VC/bin/Hostx64/x64:${VSPATH}/VC/bin/Hostx86/x86:${VSPATH}/SDK/bin/10.0.15063.0/x64:${VSPATH}/VC/redist/x64/Microsoft.VC141.CRT:${VSPATH}/SDK/Redist/ucrt/DLLs/x64:${PATH}" -export LIB="${VSPATH}/VC/lib/x64:${VSPATH}/SDK/lib/10.0.15063.0/ucrt/x64:${VSPATH}/SDK/lib/10.0.15063.0/um/x64" |