diff options
Diffstat (limited to 'taskcluster/scripts/misc')
| -rwxr-xr-x | taskcluster/scripts/misc/build-binutils-linux.sh | 16 | ||||
| -rwxr-xr-x | taskcluster/scripts/misc/build-cctools.sh | 82 | ||||
| -rwxr-xr-x | taskcluster/scripts/misc/build-clang-linux.sh | 30 | ||||
| -rwxr-xr-x | taskcluster/scripts/misc/build-clang-windows.sh | 61 | ||||
| -rwxr-xr-x | taskcluster/scripts/misc/build-gcc-linux.sh | 16 | ||||
| -rwxr-xr-x | taskcluster/scripts/misc/minidump_stackwalk.sh | 125 | ||||
| -rwxr-xr-x | taskcluster/scripts/misc/repackage-jdk-centos.sh | 45 |
7 files changed, 375 insertions, 0 deletions
diff --git a/taskcluster/scripts/misc/build-binutils-linux.sh b/taskcluster/scripts/misc/build-binutils-linux.sh new file mode 100755 index 000000000..da0eb2724 --- /dev/null +++ b/taskcluster/scripts/misc/build-binutils-linux.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -x -e -v + +# This script is for building binutils for Linux. + +WORKSPACE=$HOME/workspace +HOME_DIR=$WORKSPACE/build +UPLOAD_DIR=$WORKSPACE/artifacts + +cd $HOME_DIR/src + +build/unix/build-binutils/build-binutils.sh $HOME_DIR + +# Put a tarball in the artifacts dir +mkdir -p $UPLOAD_DIR +cp $HOME_DIR/binutils.tar.* $UPLOAD_DIR diff --git a/taskcluster/scripts/misc/build-cctools.sh b/taskcluster/scripts/misc/build-cctools.sh new file mode 100755 index 000000000..3eea0929d --- /dev/null +++ b/taskcluster/scripts/misc/build-cctools.sh @@ -0,0 +1,82 @@ +#!/bin/bash +set -x -e -v + +# This script is for building cctools (Apple's binutils) for Linux using +# crosstool-ng (https://github.com/diorcety/crosstool-ng). + +WORKSPACE=$HOME/workspace +UPLOAD_DIR=$WORKSPACE/artifacts + +# Repository info +: CROSSTOOL_NG_REPOSITORY ${CROSSTOOL_NG_REPOSITORY:=https://github.com/diorcety/crosstool-ng} +: CROSSTOOL_NG_REV ${CROSSTOOL_NG_REV:=master} + +# hacky +ln -s `which gcc` ~/bin/x86_64-linux-gnu-gcc +export PATH=$PATH:~/bin + +# Set some crosstools-ng directories +CT_TOP_DIR=$WORKSPACE/crosstool-ng-build +CT_PREFIX_DIR=$WORKSPACE/cctools +CT_SRC_DIR=$CT_TOP_DIR/src +CT_TARBALLS_DIR=$CT_TOP_DIR +CT_WORK_DIR=$CT_SRC_DIR +CT_LIB_DIR=$WORKSPACE/crosstool-ng +CT_BUILD_DIR=$CT_TOP_DIR/build +CT_LLVM_DIR=$WORKSPACE/clang +CT_BUILDTOOLS_PREFIX_DIR=$CT_PREFIX_DIR + +# Create our directories +rm -rf $CT_TOP_DIR +mkdir $CT_TOP_DIR +rm -rf $CT_PREFIX_DIR +mkdir $CT_PREFIX_DIR +mkdir -p $CT_SRC_DIR + +# Clone the crosstool-ng repo +tc-vcs checkout $CT_LIB_DIR $CROSSTOOL_NG_REPOSITORY $CROSSTOOL_NG_REPOSITORY $CROSSTOOL_NG_REV + +# Fetch clang from tooltool +cd $WORKSPACE +wget -O tooltool.py https://raw.githubusercontent.com/mozilla/build-tooltool/master/tooltool.py +chmod +x tooltool.py +: TOOLTOOL_CACHE ${TOOLTOOL_CACHE:=/home/worker/tooltool-cache} +export TOOLTOOL_CACHE + +wget ${GECKO_HEAD_REPOSITORY}/raw-file/${GECKO_HEAD_REV}/browser/config/tooltool-manifests/linux64/clang.manifest + +python tooltool.py -v --manifest=clang.manifest fetch + +# Copy clang into the crosstools-ng srcdir +cp -Rp $CT_LLVM_DIR $CT_SRC_DIR + +# Configure crosstools-ng +sed=sed +CT_CONNECT_TIMEOUT=5 +CT_BINUTILS_VERSION=809 +CT_PATCH_ORDER=bundled +CT_BUILD=x86_64-linux-gnu +CT_HOST=x86_64-linux-gnu +CT_TARGET=x86_64-apple-darwin10 +CT_LLVM_FULLNAME=clang + +cd $CT_TOP_DIR + +# gets a bit too verbose here +set +x + +. $CT_LIB_DIR/scripts/functions +. $CT_LIB_DIR/scripts/build/binutils/cctools.sh + +# Build cctools +do_binutils_get +do_binutils_extract +do_binutils_for_host + +set -x + +strip $CT_PREFIX_DIR/bin/* + +# Put a tarball in the artifacts dir +mkdir -p $UPLOAD_DIR +tar czf $UPLOAD_DIR/cctools.tar.gz -C $WORKSPACE `basename $CT_PREFIX_DIR` diff --git a/taskcluster/scripts/misc/build-clang-linux.sh b/taskcluster/scripts/misc/build-clang-linux.sh new file mode 100755 index 000000000..e1c6f2f0d --- /dev/null +++ b/taskcluster/scripts/misc/build-clang-linux.sh @@ -0,0 +1,30 @@ +#!/bin/bash +set -x -e -v + +# This script is for building clang for Linux. + +WORKSPACE=$HOME/workspace +HOME_DIR=$WORKSPACE/build +UPLOAD_DIR=$WORKSPACE/artifacts + +# Fetch our toolchain from tooltool +cd $HOME_DIR +wget -O tooltool.py https://raw.githubusercontent.com/mozilla/build-tooltool/master/tooltool.py +chmod +x tooltool.py +: TOOLTOOL_CACHE ${TOOLTOOL_CACHE:=/home/worker/tooltool-cache} +export TOOLTOOL_CACHE +cd src +$HOME_DIR/tooltool.py -m browser/config/tooltool-manifests/linux64/releng.manifest fetch + +# gets a bit too verbose here +set +x + +cd build/build-clang +# |mach python| sets up a virtualenv for us! +../../mach python ./build-clang.py -c clang-static-analysis-linux64.json + +set -x + +# Put a tarball in the artifacts dir +mkdir -p $UPLOAD_DIR +cp clang.tar.* $UPLOAD_DIR diff --git a/taskcluster/scripts/misc/build-clang-windows.sh b/taskcluster/scripts/misc/build-clang-windows.sh new file mode 100755 index 000000000..6d2acaa03 --- /dev/null +++ b/taskcluster/scripts/misc/build-clang-windows.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +set -x -e -v + +# This script is for building clang-cl on Windows. + +# Fetch our toolchain from tooltool. +wget -O tooltool.py ${TOOLTOOL_REPO}/raw/${TOOLTOOL_REV}/tooltool.py +chmod +x tooltool.py +: TOOLTOOL_CACHE ${TOOLTOOL_CACHE:=/home/worker/tooltool-cache} +export TOOLTOOL_CACHE + +TOOLTOOL_AUTH_FILE=/c/builds/relengapi.tok +if [ ! -e ${TOOLTOOL_AUTH_FILE} ]; then + echo cannot find ${TOOLTOOL_AUTH_FILE} + exit 1 +fi + +TOOLTOOL_MANIFEST=build/src/browser/config/tooltool-manifests/win32/build-clang-cl.manifest +./tooltool.py --authentication-file="${TOOLTOOL_AUTH_FILE}" -m "${TOOLTOOL_MANIFEST}" fetch + +# Set up all the Visual Studio paths. +MSVC_DIR=vs2015u3 +VSWINPATH="$(cd ${MSVC_DIR} && pwd)" + +echo vswinpath ${VSWINPATH} + +export WINDOWSSDKDIR="${VSWINPATH}/SDK" +export WIN32_REDIST_DIR="${VSWINPATH}/VC/redist/x86/Microsoft.VC140.CRT" +export WIN_UCRT_REDIST_DIR="${VSWINPATH}/SDK/Redist/ucrt/DLLs/x86" + +export PATH="${VSWINPATH}/VC/bin/amd64_x86:${VSWINPATH}/VC/bin/amd64:${VSWINPATH}/VC/bin:${VSWINPATH}/SDK/bin/x86:${VSWINPATH}/SDK/bin/x64:${VSWINPATH}/DIA SDK/bin:${PATH}" +export PATH="${VSWINPATH}/VC/redist/x86/Microsoft.VC140.CRT:${VSWINPATH}/VC/redist/x64/Microsoft.VC140.CRT:${VSWINPATH}/SDK/Redist/ucrt/DLLs/x86:${VSWINPATH}/SDK/Redist/ucrt/DLLs/x64:${PATH}" + +export INCLUDE="${VSWINPATH}/VC/include:${VSWINPATH}/VC/atlmfc/include:${VSWINPATH}/SDK/Include/10.0.14393.0/ucrt:${VSWINPATH}/SDK/Include/10.0.14393.0/shared:${VSWINPATH}/SDK/Include/10.0.14393.0/um:${VSWINPATH}/SDK/Include/10.0.14393.0/winrt:${VSWINPATH}/DIA SDK/include" +export LIB="${VSWINPATH}/VC/lib:${VSWINPATH}/VC/atlmfc/lib:${VSWINPATH}/SDK/lib/10.0.14393.0/ucrt/x86:${VSWINPATH}/SDK/lib/10.0.14393.0/um/x86:${VSWINPATH}/DIA SDK/lib" + +export PATH="$(cd svn && pwd)/bin:${PATH}" +export PATH="$(cd cmake && pwd)/bin:${PATH}" +export PATH="$(cd ninja && pwd)/bin:${PATH}" + +# We use |mach python| to set up a virtualenv automatically for us. We create +# a dummy mozconfig, because the default machinery for config.guess-choosing +# of the objdir doesn't work very well. +MOZCONFIG="$(pwd)/mozconfig" +cat > ${MOZCONFIG} <<EOF +mk_add_options MOZ_OBJDIR=$(pwd)/objdir +EOF + +# gets a bit too verbose here +set +x + +BUILD_CLANG_DIR=build/src/build/build-clang +MOZCONFIG=${MOZCONFIG} build/src/mach python ${BUILD_CLANG_DIR}/build-clang.py -c ${BUILD_CLANG_DIR}/clang-static-analysis-win32.json + +set -x + +# Put a tarball in the artifacts dir +UPLOAD_PATH=public/build +mkdir -p ${UPLOAD_PATH} +cp clang.tar.* ${UPLOAD_PATH} diff --git a/taskcluster/scripts/misc/build-gcc-linux.sh b/taskcluster/scripts/misc/build-gcc-linux.sh new file mode 100755 index 000000000..7621ec4aa --- /dev/null +++ b/taskcluster/scripts/misc/build-gcc-linux.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -e + +# This script is for building GCC for Linux. + +WORKSPACE=$HOME/workspace +HOME_DIR=$WORKSPACE/build +UPLOAD_DIR=$WORKSPACE/artifacts + +cd $HOME_DIR/src + +build/unix/build-gcc/build-gcc.sh $HOME_DIR + +# Put a tarball in the artifacts dir +mkdir -p $UPLOAD_DIR +cp $HOME_DIR/gcc.tar.* $UPLOAD_DIR diff --git a/taskcluster/scripts/misc/minidump_stackwalk.sh b/taskcluster/scripts/misc/minidump_stackwalk.sh new file mode 100755 index 000000000..de4fd748c --- /dev/null +++ b/taskcluster/scripts/misc/minidump_stackwalk.sh @@ -0,0 +1,125 @@ +#!/bin/bash +# +# This script builds minidump_stackwalk binaries from the Google Breakpad +# source for all of the operating systems that we run Firefox tests on: +# Linux x86, Linux x86-64, Windows x86, OS X x86-64. +# +# It expects to be run in the luser/breakpad-builder:0.7 Docker image and +# needs access to the relengapiproxy to download internal tooltool files. + +set -v -e -x + +# This is a pain to support properly with gclient. +#: BREAKPAD_REPO ${BREAKPAD_REPO:=https://google-breakpad.googlecode.com/svn/trunk/} +: BREAKPAD_REV "${BREAKPAD_REV:=master}" +: STACKWALK_HTTP_REPO "${STACKWALK_HTTP_REPO:=https://hg.mozilla.org/users/tmielczarek_mozilla.com/stackwalk-http}" +: STACKWALK_HTTP_REV "${STACKWALK_HTTP_REV:=default}" + +ncpu=$(getconf _NPROCESSORS_ONLN) + +function build() +{ + cd /tmp + local platform=$1 + local strip_prefix=$2 + local configure_args=$3 + local make_args=$4 + local objdir=/tmp/obj-breakpad-$platform + local ext= + if test "$platform" = "win32"; then + ext=.exe + fi + rm -rf "$objdir" + mkdir "$objdir" + # First, build Breakpad + cd "$objdir" + # shellcheck disable=SC2086 + CFLAGS="-O2 $CFLAGS" CXXFLAGS="-O2 $CXXFLAGS" /tmp/breakpad/src/configure --disable-tools $configure_args + # shellcheck disable=SC2086 + make -j$ncpu $make_args src/libbreakpad.a src/third_party/libdisasm/libdisasm.a src/processor/stackwalk_common.o + # Second, build stackwalk-http + make -f /tmp/stackwalk-http/Makefile BREAKPAD_SRCDIR=/tmp/breakpad/src "BREAKPAD_OBJDIR=$(pwd)" "OS=$platform" "-j$ncpu" + "${strip_prefix}strip" "stackwalk${ext}" + cp "stackwalk${ext}" "/tmp/stackwalker/${platform}-minidump_stackwalk${ext}" +} + +function linux64() +{ + export LDFLAGS="-static-libgcc -static-libstdc++" + build linux64 + unset LDFLAGS +} + +function linux32() +{ + export LDFLAGS="-static-libgcc -static-libstdc++ -L/tmp/libcurl-i386/lib" + export CFLAGS="-m32 -I/tmp/libcurl-i386/include" + export CXXFLAGS="-m32 -I/tmp/libcurl-i386/include" + build linux32 "" "--enable-m32" + unset LDFLAGS CFLAGS CXXFLAGS +} + +function macosx64() +{ + cd /tmp + if ! test -d MacOSX10.7.sdk; then + python tooltool.py -v --manifest=macosx-sdk.manifest --url=http://relengapi/tooltool/ fetch + fi + export MACOSX_SDK=/tmp/MacOSX10.7.sdk + export CCTOOLS=/tmp/cctools + local FLAGS="-stdlib=libc++ -target x86_64-apple-darwin10 -mlinker-version=136 -B /tmp/cctools/bin -isysroot ${MACOSX_SDK} -mmacosx-version-min=10.7" + export CC="clang $FLAGS" + export CXX="clang++ $FLAGS -std=c++11" + local old_path="$PATH" + export PATH="/tmp/clang/bin:/tmp/cctools/bin/:$PATH" + export LD_LIBRARY_PATH=/usr/lib/llvm-3.6/lib/ + + build macosx64 "/tmp/cctools/bin/x86_64-apple-darwin10-" "--host=x86_64-apple-darwin10" "AR=/tmp/cctools/bin/x86_64-apple-darwin10-ar" + + unset CC CXX LD_LIBRARY_PATH MACOSX_SDK CCTOOLS + export PATH="$old_path" +} + +function win32() +{ + export LDFLAGS="-static-libgcc -static-libstdc++" + export CFLAGS="-D__USE_MINGW_ANSI_STDIO=1" + export CXXFLAGS="-D__USE_MINGW_ANSI_STDIO=1" + export ZLIB_DIR=/tmp/zlib-mingw + build win32 "i686-w64-mingw32-" "--host=i686-w64-mingw32" + unset LDFLAGS CFLAGS CXXFLAGS ZLIB_DIR +} + +cd /tmp +if ! test -d depot_tools; then + git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git +else + (cd depot_tools; git pull origin master) +fi +export PATH=$(pwd)/depot_tools:"$PATH" +if ! test -d breakpad; then + mkdir breakpad + pushd breakpad + fetch breakpad + popd +else + pushd breakpad/src + git pull origin master + popd +fi +pushd breakpad/src +git checkout "${BREAKPAD_REV}" +gclient sync +popd + +(cd breakpad/src; git rev-parse master) +if ! test -d stackwalk-http; then + hg clone -u "$STACKWALK_HTTP_REV" "$STACKWALK_HTTP_REPO" +else + (cd stackwalk-http && hg pull "$STACKWALK_HTTP_REPO" && hg up "$STACKWALK_HTTP_REV") +fi +mkdir -p stackwalker +linux64 +linux32 +macosx64 +win32 diff --git a/taskcluster/scripts/misc/repackage-jdk-centos.sh b/taskcluster/scripts/misc/repackage-jdk-centos.sh new file mode 100755 index 000000000..2c952602b --- /dev/null +++ b/taskcluster/scripts/misc/repackage-jdk-centos.sh @@ -0,0 +1,45 @@ +#! /bin/bash + +set -e -x + +mkdir -p artifacts +pushd build + +rm -rf root && mkdir root && cd root + +# change these variables when updating java version +mirror_url_base="http://mirror.centos.org/centos/6.7/updates/x86_64/Packages" +openjdk=java-1.7.0-openjdk-1.7.0.85-2.6.1.3.el6_7.x86_64.rpm +openjdk_devel=java-1.7.0-openjdk-devel-1.7.0.85-2.6.1.3.el6_7.x86_64.rpm +jvm_openjdk_dir=java-1.7.0-openjdk-1.7.0.85.x86_64 + +# grab the rpm and unpack it +wget ${mirror_url_base}/${openjdk} +wget ${mirror_url_base}/${openjdk_devel} +rpm2cpio $openjdk | cpio -ivd +rpm2cpio $openjdk_devel | cpio -ivd + +cd usr/lib/jvm +mv $jvm_openjdk_dir java_home + +# cacerts is a relative symlink, which doesn't work when we repackage. Make it +# absolute. We could use tar's --dereference option, but there's a subtle +# difference between making the symlink absolute and using --dereference. +# Making the symlink absolute lets the consuming system set the cacerts; using +# --dereference takes the producing system's cacerts and sets them in stone. We +# prefer the flexibility of the former. +rm java_home/jre/lib/security/cacerts +ln -s /etc/pki/java/cacerts java_home/jre/lib/security/cacerts + +# document version this is based on +echo "Built from ${mirror_url_Base} + ${openjdk} + ${openjdk_devel} + +Run through rpm2cpio | cpio, and /usr/lib/jvm/${jvm_openjdk_dir} renamed to 'java_home'." > java_home/VERSION + +# tarball the unpacked rpm and put it in the taskcluster upload artifacts dir +tar -Jvcf java_home-${jvm_openjdk_dir}.tar.xz java_home +popd + +mv build/root/usr/lib/jvm/java_home-${jvm_openjdk_dir}.tar.xz artifacts |