diff options
Diffstat (limited to 'security/nss/automation/taskcluster/scripts')
17 files changed, 782 insertions, 0 deletions
diff --git a/security/nss/automation/taskcluster/scripts/build.sh b/security/nss/automation/taskcluster/scripts/build.sh new file mode 100755 index 0000000000..42ac822f28 --- /dev/null +++ b/security/nss/automation/taskcluster/scripts/build.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +source $(dirname "$0")/tools.sh + +if [ -n "$NSS_BUILD_MODULAR" ]; then + $(dirname "$0")/build_nspr.sh || exit $? + $(dirname "$0")/build_util.sh || exit $? + $(dirname "$0")/build_softoken.sh || exit $? + $(dirname "$0")/build_nss.sh || exit $? + exit +fi + +# Clone NSPR if needed. +hg_clone https://hg.mozilla.org/projects/nspr ./nspr default + +if [[ -f nss/nspr.patch && "$ALLOW_NSPR_PATCH" == "1" ]]; then + pushd nspr + cat ../nss/nspr.patch | patch -p1 + popd +fi + +# Build. +make -C nss nss_build_all + +# Package. +mkdir artifacts +tar cvfjh artifacts/dist.tar.bz2 dist diff --git a/security/nss/automation/taskcluster/scripts/build_gyp.sh b/security/nss/automation/taskcluster/scripts/build_gyp.sh new file mode 100755 index 0000000000..d8dee1f989 --- /dev/null +++ b/security/nss/automation/taskcluster/scripts/build_gyp.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +source $(dirname "$0")/tools.sh + +# Clone NSPR if needed. +hg_clone https://hg.mozilla.org/projects/nspr ./nspr default + +if [[ -f nss/nspr.patch && "$ALLOW_NSPR_PATCH" == "1" ]]; then + pushd nspr + cat ../nss/nspr.patch | patch -p1 + popd +fi + +# Dependencies +# For MacOS we have hardware in the CI which doesn't allow us o deploy VMs. +# The setup is hardcoded and can't be changed easily. +# This part is a helper We install dependencies manually to help. +if [ "$(uname)" = "Darwin" ]; then + python3 -m pip install --user gyp-next + python3 -m pip install --user ninja + export PATH="$(python3 -m site --user-base)/bin:${PATH}" +fi + +# Build. +nss/build.sh -g -v --enable-libpkix -Denable_draft_hpke=1 "$@" + +# Package. +if [[ $(uname) = "Darwin" ]]; then + mkdir -p public + tar cvfjh public/dist.tar.bz2 dist +else + mkdir artifacts + tar cvfjh artifacts/dist.tar.bz2 dist +fi diff --git a/security/nss/automation/taskcluster/scripts/build_image.sh b/security/nss/automation/taskcluster/scripts/build_image.sh new file mode 100755 index 0000000000..3a469cf224 --- /dev/null +++ b/security/nss/automation/taskcluster/scripts/build_image.sh @@ -0,0 +1,27 @@ +#!/bin/bash -vex + +set -x -e -v + +# Prefix errors with taskcluster error prefix so that they are parsed by Treeherder +raise_error() { + echo + echo "[taskcluster-image-build:error] $1" + exit 1 +} + +# Ensure that the PROJECT is specified so the image can be indexed +test -n "$PROJECT" || raise_error "Project must be provided." +test -n "$HASH" || raise_error "Context Hash must be provided." + +CONTEXT_PATH="/home/worker/nss/$CONTEXT_PATH" + +test -d "$CONTEXT_PATH" || raise_error "Context Path $CONTEXT_PATH does not exist." +test -f "$CONTEXT_PATH/Dockerfile" || raise_error "Dockerfile must be present in $CONTEXT_PATH." + +apt-get update +apt-get -y install zstd + +docker build -t "$PROJECT:$HASH" "$CONTEXT_PATH" + +mkdir /artifacts +docker save "$PROJECT:$HASH" | zstd > /artifacts/image.tar.zst diff --git a/security/nss/automation/taskcluster/scripts/build_nspr.sh b/security/nss/automation/taskcluster/scripts/build_nspr.sh new file mode 100755 index 0000000000..b104e430a4 --- /dev/null +++ b/security/nss/automation/taskcluster/scripts/build_nspr.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -v -e -x + +source $(dirname $0)/tools.sh + +# Clone NSPR if needed. +hg_clone https://hg.mozilla.org/projects/nspr nspr default + +if [[ -f nss/nspr.patch && "$ALLOW_NSPR_PATCH" == "1" ]]; then + pushd nspr + cat ../nss/nspr.patch | patch -p1 + popd +fi + +# Build. +rm -rf dist +make -C nss build_nspr + +# Package. +test -d artifacts || mkdir artifacts +rm -rf dist-nspr +mv dist dist-nspr +tar cvfjh artifacts/dist-nspr.tar.bz2 dist-nspr diff --git a/security/nss/automation/taskcluster/scripts/build_nss.sh b/security/nss/automation/taskcluster/scripts/build_nss.sh new file mode 100755 index 0000000000..b909bc30ed --- /dev/null +++ b/security/nss/automation/taskcluster/scripts/build_nss.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +set -v -e -x + +source $(dirname $0)/tools.sh +source $(dirname $0)/split.sh + +test -d dist-softoken || { echo "run build_softoken.sh first" 1>&2; exit 1; } + +rm -rf nss-nss +split_nss nss nss-nss + +# Build. +export NSS_BUILD_WITHOUT_SOFTOKEN=1 +export NSS_USE_SYSTEM_FREEBL=1 + +platform=`make -s -C nss platform` + +export NSPR_LIB_DIR="$PWD/dist-nspr/$platform/lib" +export NSSUTIL_LIB_DIR="$PWD/dist-util/$platform/lib" +export FREEBL_LIB_DIR="$PWD/dist-softoken/$platform/lib" +export SOFTOKEN_LIB_DIR="$PWD/dist-softoken/$platform/lib" +export FREEBL_LIBS=-lfreebl + +export NSS_NO_PKCS11_BYPASS=1 +export FREEBL_NO_DEPEND=1 + +export LIBRARY_PATH="$PWD/dist-nspr/$platform/lib:$PWD/dist-util/$platform/lib:$PWD/dist-softoken/$platform/lib" +export LD_LIBRARY_PATH="$LIBRARY_PATH:$LD_LIBRARY_PATH" +export INCLUDES="-I$PWD/dist-nspr/$platform/include -I$PWD/dist-util/public/nss -I$PWD/dist-softoken/public/nss" + +rm -rf dist +make -C nss-nss nss_build_all + +# Package. +test -d artifacts || mkdir artifacts +rm -rf dist-nss +mv dist dist-nss +tar cvfjh artifacts/dist-nss.tar.bz2 dist-nss diff --git a/security/nss/automation/taskcluster/scripts/build_softoken.sh b/security/nss/automation/taskcluster/scripts/build_softoken.sh new file mode 100755 index 0000000000..5f60456022 --- /dev/null +++ b/security/nss/automation/taskcluster/scripts/build_softoken.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash + +set -v -e -x + +source $(dirname $0)/tools.sh +source $(dirname $0)/split.sh + +test -d dist-util || { echo "run build_util.sh first" 1>&2; exit 1; } + +rm -rf nss-softoken +split_softoken nss nss-softoken + +# Build. +platform=`make -s -C nss platform` +export LIBRARY_PATH="$PWD/dist-nspr/$platform/lib:$PWD/dist-util/$platform/lib" +export LD_LIBRARY_PATH="$LIBRARY_PATH:$LD_LIBRARY_PATH" +export INCLUDES="-I$PWD/dist-nspr/$platform/include -I$PWD/dist-util/public/nss" +export NSS_BUILD_SOFTOKEN_ONLY=1 + +rm -rf dist +make -C nss-softoken nss_build_all + +for i in blapi alghmac cmac; do + mv "dist/private/nss/${i}.h" dist/public/nss +done + +# Package. +test -d artifacts || mkdir artifacts +rm -rf dist-softoken +mv dist dist-softoken +tar cvfjh artifacts/dist-softoken.tar.bz2 dist-softoken diff --git a/security/nss/automation/taskcluster/scripts/build_util.sh b/security/nss/automation/taskcluster/scripts/build_util.sh new file mode 100755 index 0000000000..0d2ecc5e87 --- /dev/null +++ b/security/nss/automation/taskcluster/scripts/build_util.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +set -v -e -x + +source $(dirname $0)/tools.sh +source $(dirname $0)/split.sh + +rm -rf nss-util +split_util nss nss-util + +# Build. +platform=`make -s -C nss platform` +export LIBRARY_PATH="$PWD/dist-nspr/$platform/lib" +export LD_LIBRARY_PATH="$LIBRARY_PATH:$LD_LIBRARY_PATH" +export INCLUDES="-I$PWD/dist-nspr/$platform/include" +export NSS_BUILD_UTIL_ONLY=1 + +rm -rf dist +make -C nss-util nss_build_all + +# Package. +test -d artifacts || mkdir artifacts +rm -rf dist-util +mv dist dist-util +tar cvfjh artifacts/dist-util.tar.bz2 dist-util diff --git a/security/nss/automation/taskcluster/scripts/check_abi.sh b/security/nss/automation/taskcluster/scripts/check_abi.sh new file mode 100755 index 0000000000..0bd8d9693b --- /dev/null +++ b/security/nss/automation/taskcluster/scripts/check_abi.sh @@ -0,0 +1,180 @@ +#! /bin/bash + +set_env() +{ + cd /home/worker + HGDIR=/home/worker + OUTPUTDIR=$(pwd)$(echo "/output") + DATE=$(date "+TB [%Y-%m-%d %H:%M:%S]") + + if [ ! -d "${OUTPUTDIR}" ]; then + echo "Creating output dir" + mkdir "${OUTPUTDIR}" + fi + + if [ ! -d "nspr" ]; then + for i in 0 2 5; do + sleep $i + hg clone -r "default" "https://hg.mozilla.org/projects/nspr" "${HGDIR}/nspr" && break + rm -rf nspr + done + fi + + if [[ -f nss/nspr.patch && "$ALLOW_NSPR_PATCH" == "1" ]]; then + pushd nspr + cat ../nss/nspr.patch | patch -p1 + popd + fi + + cd nss + ./build.sh -v -c + cd .. +} + +check_abi() +{ + set_env + set +e #reverses set -e from build.sh to allow possible hg clone failures + if [[ "$1" != --nobuild ]]; then # Start nobuild block + + echo "######## NSS ABI CHECK ########" + echo "######## creating temporary HG clones ########" + + rm -rf ${HGDIR}/baseline + mkdir ${HGDIR}/baseline + BASE_NSS=`cat ${HGDIR}/nss/automation/abi-check/previous-nss-release` #Reads the version number of the last release from the respective file + NSS_CLONE_RESULT=0 + for i in 0 2 5; do + sleep $i + hg clone -u "${BASE_NSS}" "https://hg.mozilla.org/projects/nss" "${HGDIR}/baseline/nss" + if [ $? -eq 0 ]; then + NSS_CLONE_RESULT=0 + break + fi + rm -rf "${HGDIR}/baseline/nss" + NSS_CLONE_RESULT=1 + done + if [ ${NSS_CLONE_RESULT} -ne 0 ]; then + echo "invalid tag in automation/abi-check/previous-nss-release" + return 1 + fi + + BASE_NSPR=NSPR_$(head -1 ${HGDIR}/baseline/nss/automation/release/nspr-version.txt | cut -d . -f 1-2 | tr . _)_BRANCH + hg clone -u "${BASE_NSPR}" "https://hg.mozilla.org/projects/nspr" "${HGDIR}/baseline/nspr" + NSPR_CLONE_RESULT=$? + + if [ ${NSPR_CLONE_RESULT} -ne 0 ]; then + rm -rf "${HGDIR}/baseline/nspr" + for i in 0 2 5; do + sleep $i + hg clone -u "default" "https://hg.mozilla.org/projects/nspr" "${HGDIR}/baseline/nspr" && break + rm -rf "${HGDIR}/baseline/nspr" + done + echo "Nonexisting tag ${BASE_NSPR} derived from ${BASE_NSS} automation/release/nspr-version.txt" + echo "Using default branch instead." + fi + + echo "######## building baseline NSPR/NSS ########" + echo "${HGDIR}/baseline/nss/build.sh" + cd ${HGDIR}/baseline/nss + ./build.sh -v -c + cd ${HGDIR} + else # Else nobuild block + echo "######## using existing baseline NSPR/NSS build ########" + fi # End nobuild block + + set +e #reverses set -e from build.sh to allow abidiff failures + + echo "######## Starting abidiff procedure ########" + abi_diff +} + +#Slightly modified from build.sh in this directory +abi_diff() +{ + ABI_PROBLEM_FOUND=0 + ABI_REPORT=${OUTPUTDIR}/abi-diff.txt + rm -f ${ABI_REPORT} + PREVDIST=${HGDIR}/baseline/dist + NEWDIST=${HGDIR}/dist + # libnssdbm3.so isn't built by default anymore, skip it. + ALL_SOs="libfreebl3.so libfreeblpriv3.so libnspr4.so libnss3.so libnssckbi.so libnsssysinit.so libnssutil3.so libplc4.so libplds4.so libsmime3.so libsoftokn3.so libssl3.so" + for SO in ${ALL_SOs}; do + if [ ! -f ${HGDIR}/nss/automation/abi-check/expected-report-$SO.txt ]; then + touch ${HGDIR}/nss/automation/abi-check/expected-report-$SO.txt + fi + abidiff --hd1 $PREVDIST/public/ --hd2 $NEWDIST/public \ + $PREVDIST/*/lib/$SO $NEWDIST/*/lib/$SO \ + > ${HGDIR}/nss/automation/abi-check/new-report-temp$SO.txt + RET=$? + cat ${HGDIR}/nss/automation/abi-check/new-report-temp$SO.txt \ + | grep -v "^Functions changes summary:" \ + | grep -v "^Variables changes summary:" \ + | sed -e 's/__anonymous_enum__[0-9]*/__anonymous_enum__/g' \ + > ${HGDIR}/nss/automation/abi-check/new-report-$SO.txt + rm -f ${HGDIR}/nss/automation/abi-check/new-report-temp$SO.txt + + ABIDIFF_ERROR=$((($RET & 0x01) != 0)) + ABIDIFF_USAGE_ERROR=$((($RET & 0x02) != 0)) + ABIDIFF_ABI_CHANGE=$((($RET & 0x04) != 0)) + ABIDIFF_ABI_INCOMPATIBLE_CHANGE=$((($RET & 0x08) != 0)) + ABIDIFF_UNKNOWN_BIT_SET=$((($RET & 0xf0) != 0)) + + # If abidiff reports an error, or a usage error, or if it sets a result + # bit value this script doesn't know yet about, we'll report failure. + # For ABI changes, we don't yet report an error. We'll compare the + # result report with our allowlist. This allows us to silence changes + # that we're already aware of and have been declared acceptable. + + REPORT_RET_AS_FAILURE=0 + if [ $ABIDIFF_ERROR -ne 0 ]; then + echo "abidiff reported ABIDIFF_ERROR." + REPORT_RET_AS_FAILURE=1 + fi + if [ $ABIDIFF_USAGE_ERROR -ne 0 ]; then + echo "abidiff reported ABIDIFF_USAGE_ERROR." + REPORT_RET_AS_FAILURE=1 + fi + if [ $ABIDIFF_UNKNOWN_BIT_SET -ne 0 ]; then + echo "abidiff reported ABIDIFF_UNKNOWN_BIT_SET." + REPORT_RET_AS_FAILURE=1 + fi + + if [ $ABIDIFF_ABI_CHANGE -ne 0 ]; then + echo "Ignoring abidiff result ABI_CHANGE, instead we'll check for non-allowlisted differences." + fi + if [ $ABIDIFF_ABI_INCOMPATIBLE_CHANGE -ne 0 ]; then + echo "Ignoring abidiff result ABIDIFF_ABI_INCOMPATIBLE_CHANGE, instead we'll check for non-allowlisted differences." + fi + + if [ $REPORT_RET_AS_FAILURE -ne 0 ]; then + ABI_PROBLEM_FOUND=1 + echo "abidiff {$PREVDIST , $NEWDIST} for $SO FAILED with result $RET, or failed writing to ${HGDIR}/nss/automation/abi-check/new-report-$SO.txt" + fi + if [ ! -f ${HGDIR}/nss/automation/abi-check/expected-report-$SO.txt ]; then + ABI_PROBLEM_FOUND=1 + echo "FAILED to access report file: ${HGDIR}/nss/automation/abi-check/expected-report-$SO.txt" + fi + + diff -wB -u ${HGDIR}/nss/automation/abi-check/expected-report-$SO.txt \ + ${HGDIR}/nss/automation/abi-check/new-report-$SO.txt >> ${ABI_REPORT} + if [ ! -f ${ABI_REPORT} ]; then + ABI_PROBLEM_FOUND=1 + echo "FAILED to compare exepcted and new report: ${HGDIR}/nss/automation/abi-check/new-report-$SO.txt" + fi + done + + if [ -s ${ABI_REPORT} ]; then + echo "FAILED: there are new unexpected ABI changes" + cat ${ABI_REPORT} + return 1 + elif [ $ABI_PROBLEM_FOUND -ne 0 ]; then + echo "FAILED: failure executing the ABI checks" + cat ${ABI_REPORT} + return 1 + fi + + return 0 +} + +check_abi $1 diff --git a/security/nss/automation/taskcluster/scripts/extend_task_graph.sh b/security/nss/automation/taskcluster/scripts/extend_task_graph.sh new file mode 100755 index 0000000000..ade84cd602 --- /dev/null +++ b/security/nss/automation/taskcluster/scripts/extend_task_graph.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +source $(dirname "$0")/tools.sh + +mkdir -p /home/worker/artifacts + +# Install Node.JS dependencies. +cd nss/automation/taskcluster/graph/ && npm install + +# Extend the task graph. +node lib/index.js diff --git a/security/nss/automation/taskcluster/scripts/fuzz.sh b/security/nss/automation/taskcluster/scripts/fuzz.sh new file mode 100755 index 0000000000..75851ff5b9 --- /dev/null +++ b/security/nss/automation/taskcluster/scripts/fuzz.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +source $(dirname "$0")/tools.sh + +type="$1" +shift + +# Fetch artifact if needed. +fetch_dist + +# Clone corpus. +./nss/fuzz/config/clone_corpus.sh + +# Ensure we have a corpus. +if [ ! -d "nss/fuzz/corpus/$type" ]; then + mkdir -p nss/fuzz/corpus/$type + + set +x + + # Create a corpus out of what we have. + for f in $(find nss/fuzz/corpus -type f); do + cp $f "nss/fuzz/corpus/$type" + done + + set -x +fi + +# Fetch objdir name. +objdir=$(cat dist/latest) + +# Run nssfuzz. +dist/$objdir/bin/nssfuzz-"$type" "$@" diff --git a/security/nss/automation/taskcluster/scripts/gen_certs.sh b/security/nss/automation/taskcluster/scripts/gen_certs.sh new file mode 100755 index 0000000000..c03db7e9c2 --- /dev/null +++ b/security/nss/automation/taskcluster/scripts/gen_certs.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash + +source $(dirname "$0")/tools.sh + +# Fetch artifact if needed. +fetch_dist + +# Generate certificates. +NSS_TESTS=cert NSS_CYCLES="standard pkix sharedb" $(dirname $0)/run_tests.sh + +# Reset test counter so that test runs pick up our certificates. +echo 1 > tests_results/security/localhost + +# Package. +if [[ $(uname) = "Darwin" ]]; then + mkdir -p public + tar cvfjh public/dist.tar.bz2 dist tests_results +else + mkdir artifacts + tar cvfjh artifacts/dist.tar.bz2 dist tests_results +fi diff --git a/security/nss/automation/taskcluster/scripts/gen_coverage_report.sh b/security/nss/automation/taskcluster/scripts/gen_coverage_report.sh new file mode 100755 index 0000000000..dc7d77d6c1 --- /dev/null +++ b/security/nss/automation/taskcluster/scripts/gen_coverage_report.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +source $(dirname "$0")/tools.sh + +# Clone NSPR. +hg_clone https://hg.mozilla.org/projects/nspr ./nspr default + +if [[ -f nss/nspr.patch && "$ALLOW_NSPR_PATCH" == "1" ]]; then + pushd nspr + cat ../nss/nspr.patch | patch -p1 + popd +fi + +out=/home/worker/artifacts +mkdir -p $out + +# Generate coverage report. +cd nss && ./mach coverage --outdir=$out ssl_gtests diff --git a/security/nss/automation/taskcluster/scripts/run_hacl.sh b/security/nss/automation/taskcluster/scripts/run_hacl.sh new file mode 100755 index 0000000000..f9831d24fd --- /dev/null +++ b/security/nss/automation/taskcluster/scripts/run_hacl.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +if [[ $(id -u) -eq 0 ]]; then + # Drop privileges by re-running this script. + # Note: this mangles arguments, better to avoid running scripts as root. + exec su worker -c "$0 $*" +fi + +set -e -x -v + +# The docker image this is running in has NSS sources. +# Get the HACL* source, containing a snapshot of the C code, extracted on the +# HACL CI. +git clone -q "https://github.com/hacl-star/hacl-star" ~/hacl-star +git -C ~/hacl-star checkout -q 72f9d0c783cb716add714344604d591106dfbf7f + +# Format the C snapshot. +cd ~/hacl-star/dist/mozilla +cp ~/nss/.clang-format . +find . -type f -name '*.[ch]' -exec clang-format -i {} \+ +cd ~/hacl-star/dist/karamel +cp ~/nss/.clang-format . +find . -type f -name '*.[ch]' -exec clang-format -i {} \+ + +# These diff commands will return 1 if there are differences and stop the script. + +# We have two checks in the script. +# The first one only checks the files in the verified/internal folder; the second one does for all the rest +# It was implemented like this due to not uniqueness of the names in the verified folders +# For instance, the files Hacl_Chacha20.h are present in both directories, but the content differs. + +files=($(find ~/nss/lib/freebl/verified/internal -type f -name '*.[ch]')) +for f in "${files[@]}"; do + file_name=$(basename "$f") + hacl_file=($(find ~/hacl-star/dist/mozilla/internal/ -type f -name $file_name)) + diff $hacl_file $f +done + +files=($(find ~/nss/lib/freebl/verified/ -type f -name '*.[ch]' -not -path "*/freebl/verified/internal/*" -not -path "*/freebl/verified/config.h")) +for f in "${files[@]}"; do + file_name=$(basename "$f") + hacl_file=($(find ~/hacl-star/dist/mozilla/ ~/hacl-star/dist/karamel/ -type f -name $file_name -not -path "*/hacl-star/dist/mozilla/internal/*")) + # TODO(Bug 1854438): Remove P384 exception. + # TODO(Bug 1854439): Remove P521 exception. + if [ $file_name == "Hacl_P384.c" \ + -o $file_name == "Hacl_P384.h" \ + -o $file_name == "Hacl_P521.c" \ + -o $file_name == "Hacl_P521.h" ] + then + continue; + fi + diff $hacl_file $f +done diff --git a/security/nss/automation/taskcluster/scripts/run_scan_build.sh b/security/nss/automation/taskcluster/scripts/run_scan_build.sh new file mode 100755 index 0000000000..0e4fcbdf0d --- /dev/null +++ b/security/nss/automation/taskcluster/scripts/run_scan_build.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +source $(dirname "$0")/tools.sh + +# Clone NSPR if needed. +if [ ! -d "nspr" ]; then + hg_clone https://hg.mozilla.org/projects/nspr ./nspr default + + if [[ -f nss/nspr.patch && "$ALLOW_NSPR_PATCH" == "1" ]]; then + pushd nspr + cat ../nss/nspr.patch | patch -p1 + popd + fi +fi + +# Build. +cd nss +make nss_build_all + +# What we want to scan. +# key: directory to scan +# value: number of errors expected in that directory +declare -A scan=( \ + [lib/base]=0 \ + [lib/certdb]=0 \ + [lib/certhigh]=0 \ + [lib/ckfw]=0 \ + [lib/crmf]=0 \ + [lib/cryptohi]=0 \ + [lib/dev]=0 \ + [lib/freebl]=0 \ + [lib/nss]=0 \ + [lib/ssl]=0 \ + [lib/util]=0 \ + ) + +# remove .OBJ directories to force a rebuild of just the select few +for i in "${!scan[@]}"; do + find "$i" -name "*.OBJ" -exec rm -rf {} \+ +done + +# run scan-build (only building affected directories) +scan-build-5.0 -o /home/worker/artifacts --use-cc=$CC --use-c++=$CCC make nss_build_all && cd .. + +# print errors we found +set +v +x +STATUS=0 +for i in "${!scan[@]}"; do + n=$(grep -Rn "$i" /home/worker/artifacts/*/report-*.html | wc -l) + if [ $n -ne ${scan[$i]} ]; then + STATUS=1 + echo "$(date '+%T') WARNING - TEST-UNEXPECTED-FAIL: $i contains $n scan-build errors" + elif [ $n -ne 0 ]; then + echo "$(date '+%T') WARNING - TEST-EXPECTED-FAIL: $i contains $n scan-build errors" + fi +done +exit $STATUS diff --git a/security/nss/automation/taskcluster/scripts/run_tests.sh b/security/nss/automation/taskcluster/scripts/run_tests.sh new file mode 100755 index 0000000000..b8e26761a9 --- /dev/null +++ b/security/nss/automation/taskcluster/scripts/run_tests.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +source $(dirname "$0")/tools.sh + +# Fetch artifact if needed. +fetch_dist + +# Run tests. +cd nss/tests && ./all.sh diff --git a/security/nss/automation/taskcluster/scripts/split.sh b/security/nss/automation/taskcluster/scripts/split.sh new file mode 100644 index 0000000000..04fe06fb27 --- /dev/null +++ b/security/nss/automation/taskcluster/scripts/split.sh @@ -0,0 +1,148 @@ +copy_top() +{ + srcdir_="$1" + dstdir_="$2" + files=`find "$srcdir_" -maxdepth 1 -mindepth 1 -type f` + for f in $files; do + cp -p "$f" "$dstdir_" + done +} + +split_util() { + nssdir="$1" + dstdir="$2" + + # Prepare a source tree only containing files to build nss-util: + # + # nss/dbm full directory + # nss/coreconf full directory + # nss top files only + # nss/lib top files only + # nss/lib/util full directory + + # Copy everything. + cp -R $nssdir $dstdir + + # Remove subdirectories that we don't want. + rm -rf $dstdir/cmd + rm -rf $dstdir/lib + rm -rf $dstdir/automation + rm -rf $dstdir/doc + + # Start with an empty cmd lib directories to be filled selectively. + mkdir $dstdir/cmd + cp $nssdir/cmd/Makefile $dstdir/cmd + cp $nssdir/cmd/manifest.mn $dstdir/cmd + cp $nssdir/cmd/platlibs.mk $dstdir/cmd + cp $nssdir/cmd/platrules.mk $dstdir/cmd + + # Copy some files at the top and the util subdirectory recursively. + mkdir $dstdir/lib + cp $nssdir/lib/Makefile $dstdir/lib + cp -R $nssdir/lib/util $dstdir/lib/util +} + +split_softoken() { + nssdir="$1" + dstdir="$2" + + # Prepare a source tree only containing files to build nss-softoken: + # + # nss/dbm full directory + # nss/coreconf full directory + # nss top files only + # nss/lib top files only + # nss/lib/freebl full directory + # nss/lib/softoken full directory + # nss/lib/softoken/dbm full directory + + # Copy everything. + cp -R $nssdir $dstdir + + # Skip gtests when building. + sed '/^DIRS = /s/ cpputil gtests$//' $nssdir/manifest.mn > $dstdir/manifest.mn-t && mv $dstdir/manifest.mn-t $dstdir/manifest.mn + + # Remove subdirectories that we don't want. + rm -rf $dstdir/cmd + rm -rf $dstdir/tests + rm -rf $dstdir/lib + rm -rf $dstdir/pkg + rm -rf $dstdir/automation + rm -rf $dstdir/gtests + rm -rf $dstdir/cpputil + rm -rf $dstdir/doc + + # Start with an empty lib directory and copy only what we need. + mkdir $dstdir/lib + copy_top $nssdir/lib $dstdir/lib + cp -R $nssdir/lib/dbm $dstdir/lib/dbm + cp -R $nssdir/lib/freebl $dstdir/lib/freebl + cp -R $nssdir/lib/softoken $dstdir/lib/softoken + cp -R $nssdir/lib/sqlite $dstdir/lib/sqlite + + mkdir $dstdir/cmd + copy_top $nssdir/cmd $dstdir/cmd + cp -R $nssdir/cmd/bltest $dstdir/cmd/bltest + cp -R $nssdir/cmd/ecperf $dstdir/cmd/ecperf + cp -R $nssdir/cmd/dbtool $dstdir/cmd/dbtool + cp -R $nssdir/cmd/fbectest $dstdir/cmd/fbectest + cp -R $nssdir/cmd/fipstest $dstdir/cmd/fipstest + cp -R $nssdir/cmd/lib $dstdir/cmd/lib + cp -R $nssdir/cmd/lowhashtest $dstdir/cmd/lowhashtest + cp -R $nssdir/cmd/shlibsign $dstdir/cmd/shlibsign + + mkdir $dstdir/tests + copy_top $nssdir/tests $dstdir/tests + + cp -R $nssdir/tests/cipher $dstdir/tests/cipher + cp -R $nssdir/tests/common $dstdir/tests/common + cp -R $nssdir/tests/ec $dstdir/tests/ec + cp -R $nssdir/tests/lowhash $dstdir/tests/lowhash + + cp $nssdir/lib/util/verref.h $dstdir/lib/freebl + cp $nssdir/lib/util/verref.h $dstdir/lib/softoken + cp $nssdir/lib/util/verref.h $dstdir/lib/softoken/legacydb +} + +split_nss() { + nssdir="$1" + dstdir="$2" + + # Prepare a source tree only containing files to build nss: + # + # nss/dbm full directory + # nss/coreconf full directory + # nss top files only + # nss/lib top files only + # nss/lib/freebl full directory + # nss/lib/softoken full directory + # nss/lib/softoken/dbm full directory + + # Copy everything. + cp -R $nssdir $dstdir + + # Remove subdirectories that we don't want. + rm -rf $dstdir/lib/freebl + rm -rf $dstdir/lib/softoken + rm -rf $dstdir/lib/util + rm -rf $dstdir/cmd/bltest + rm -rf $dstdir/cmd/fipstest + rm -rf $dstdir/cmd/rsaperf_low + + # Copy these headers until the upstream bug is accepted + # Upstream https://bugzilla.mozilla.org/show_bug.cgi?id=820207 + cp $nssdir/lib/softoken/lowkeyi.h $dstdir/cmd/rsaperf + cp $nssdir/lib/softoken/lowkeyti.h $dstdir/cmd/rsaperf + + # Copy verref.h which will be needed later during the build phase. + cp $nssdir/lib/util/verref.h $dstdir/lib/ckfw/builtins/verref.h + cp $nssdir/lib/util/verref.h $dstdir/lib/nss/verref.h + cp $nssdir/lib/util/verref.h $dstdir/lib/smime/verref.h + cp $nssdir/lib/util/verref.h $dstdir/lib/ssl/verref.h + cp $nssdir/lib/util/templates.c $dstdir/lib/nss/templates.c + + # FIXME: Skip util_gtest because it links with libnssutil.a. Note + # that we can't use libnssutil3.so instead, because util_gtest + # depends on internal symbols not exported from the shared library. + sed '/ util_gtest \\/d' $dstdir/gtests/manifest.mn > $dstdir/gtests/manifest.mn-t && mv $dstdir/gtests/manifest.mn-t $dstdir/gtests/manifest.mn +} diff --git a/security/nss/automation/taskcluster/scripts/tools.sh b/security/nss/automation/taskcluster/scripts/tools.sh new file mode 100644 index 0000000000..81563f5066 --- /dev/null +++ b/security/nss/automation/taskcluster/scripts/tools.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash + +set -v -e -x + +# Assert that we're not running as root. +if [[ $(id -u) -eq 0 ]]; then + # This exec is still needed until aarch64 images are updated (Bug 1488325). + # Remove when images are updated. Until then, assert that things are good. + [[ $(uname -m) == aarch64 ]] + exec su worker -c "$0 $*" +fi + +export PATH="${PATH}:/home/worker/.cargo/bin/:/usr/lib/go-1.6/bin" + +# Usage: hg_clone repo dir [revision=@] +hg_clone() { + repo=$1 + dir=$2 + rev=${3:-@} + if [ -d "$dir" ]; then + hg pull -R "$dir" -ur "$rev" "$repo" && return + rm -rf "$dir" + fi + for i in 0 2 5; do + sleep $i + hg clone -r "$rev" "$repo" "$dir" && return + rm -rf "$dir" + done + exit 1 +} + +fetch_dist() { + if [ "$TASKCLUSTER_ROOT_URL" = "https://taskcluster.net" ] || [ -z "$TASKCLUSTER_ROOT_URL" ]; then + url=https://queue.taskcluster.net/v1/task/$TC_PARENT_TASK_ID/artifacts/public/dist.tar.bz2 + else + url=$TASKCLUSTER_ROOT_URL/api/queue/v1/task/$TC_PARENT_TASK_ID/artifacts/public/dist.tar.bz2 + fi + if [ ! -d "dist" ]; then + for i in 0 2 5; do + sleep $i + curl --retry 3 -Lo dist.tar.bz2 $url && tar xvjf dist.tar.bz2 && return + rm -fr dist.tar.bz2 dist + done + exit 1 + fi +} |