diff options
Diffstat (limited to 'taskcluster/scripts/builder')
| -rwxr-xr-x | taskcluster/scripts/builder/build-haz-linux.sh | 190 | ||||
| -rwxr-xr-x | taskcluster/scripts/builder/build-l10n.sh | 111 | ||||
| -rwxr-xr-x | taskcluster/scripts/builder/build-linux.sh | 130 | ||||
| -rwxr-xr-x | taskcluster/scripts/builder/build-sm-mozjs-crate.sh | 17 | ||||
| -rwxr-xr-x | taskcluster/scripts/builder/build-sm-package.sh | 39 | ||||
| -rwxr-xr-x | taskcluster/scripts/builder/build-sm-rust-bindings.sh | 17 | ||||
| -rwxr-xr-x | taskcluster/scripts/builder/build-sm.sh | 57 | ||||
| -rwxr-xr-x | taskcluster/scripts/builder/repackage.sh | 104 | ||||
| -rw-r--r-- | taskcluster/scripts/builder/requirements.txt | 1 | ||||
| -rwxr-xr-x | taskcluster/scripts/builder/sm-tooltool-config.sh | 67 |
10 files changed, 733 insertions, 0 deletions
diff --git a/taskcluster/scripts/builder/build-haz-linux.sh b/taskcluster/scripts/builder/build-haz-linux.sh new file mode 100755 index 0000000000..ad572876e6 --- /dev/null +++ b/taskcluster/scripts/builder/build-haz-linux.sh @@ -0,0 +1,190 @@ +#!/bin/bash -ex + +function usage() { + echo "Usage: $0 [--project <js|browser>] <workspace-dir> flags..." + echo "flags are treated the same way as a commit message would be" + echo "(as in, they are scanned for directives just like a try: ... line)" +} + +PROJECT=js +WORKSPACE= +while [[ $# -gt 0 ]]; do + if [[ "$1" == "-h" ]] || [[ "$1" == "--help" ]]; then + usage + exit 0 + elif [[ "$1" == "--project" ]]; then + shift + PROJECT="$1" + shift + elif [[ "$1" == "--no-tooltool" ]]; then + shift + elif [[ -z "$WORKSPACE" ]]; then + WORKSPACE=$( cd "$1" && pwd ) + shift + break + fi +done + +function check_commit_msg () { + ( set +e; + if [[ -n "$AUTOMATION" ]]; then + hg --cwd "$GECKO_PATH" log -r. --template '{desc}\n' | grep -F -q -- "$1" + else + echo -- "$SCRIPT_FLAGS" | grep -F -q -- "$1" + fi + ) +} + +if check_commit_msg "--dep"; then + HAZ_DEP=1 +fi + +SCRIPT_FLAGS=$* + +ANALYSIS_DIR="$WORKSPACE/haz-$PROJECT" + +# Ensure all the scripts in this dir are on the path.... +DIRNAME=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) +PATH=$DIRNAME:$PATH + +# Use GECKO_BASE_REPOSITORY as a signal for whether we are running in automation. +export AUTOMATION=${GECKO_BASE_REPOSITORY:+1} + +: "${GECKO_PATH:="$DIRNAME"/../../..}" + +if ! [ -d "$GECKO_PATH" ]; then + echo "GECKO_PATH must be set to a directory containing a gecko source checkout" >&2 + exit 1 +fi + +# Directory to hold the compiled JS shell that will run the analysis. +HAZARD_SHELL_OBJDIR=$WORKSPACE/obj-haz-shell + +export NO_MERCURIAL_SETUP_CHECK=1 + +if [[ "$PROJECT" = "browser" ]]; then ( + cd "$WORKSPACE" + set "$WORKSPACE" + # Mozbuild config: + export MOZBUILD_STATE_PATH=$WORKSPACE/mozbuild/ + # Create .mozbuild so mach doesn't complain about this + mkdir -p "$MOZBUILD_STATE_PATH" +) fi + +# Build the shell +export HAZARD_SHELL_OBJDIR # This will be picked up by mozconfig.haz_shell. +$GECKO_PATH/mach hazards build-shell + +# Run a self-test +$GECKO_PATH/mach hazards self-test --shell-objdir="$HAZARD_SHELL_OBJDIR" + +# Artifacts folder is outside of the cache. +mkdir -p "$HOME"/artifacts/ || true + +function grab_artifacts () { + local artifacts + artifacts="$HOME/artifacts" + + [ -d "$ANALYSIS_DIR" ] && ( + cd "$ANALYSIS_DIR" + ls -lah + + # Do not error out if no files found + shopt -s nullglob + set +e + local important + important=(refs.txt unnecessary.txt hazards.txt gcFunctions.txt allFunctions.txt heapWriteHazards.txt) + + # Bundle up the less important but still useful intermediate outputs, + # just to cut down on the clutter in treeherder's Job Details pane. + tar -acvf "${artifacts}/hazardIntermediates.tar.xz" --exclude-from <(IFS=$'\n'; echo "${important[*]}") *.txt *.lst build_xgill.log + + # Upload the important outputs individually, so that they will be + # visible in Job Details and accessible to automated jobs. + for f in "${important[@]}"; do + gzip -9 -c "$f" > "${artifacts}/$f.gz" + done + + # Check whether the user requested .xdb file upload in the top commit comment + if check_commit_msg "--upload-xdbs"; then + HAZ_UPLOAD_XDBS=1 + fi + + if [ -n "$HAZ_UPLOAD_XDBS" ]; then + for f in *.xdb; do + xz -c "$f" > "${artifacts}/$f.bz2" + done + fi + ) +} + +function check_hazards () { + ( + set +e + NUM_HAZARDS=$(grep -c 'Function.*has unrooted.*live across GC call' "$1"/rootingHazards.txt) + NUM_UNSAFE=$(grep -c '^Function.*takes unsafe address of unrooted' "$1"/refs.txt) + NUM_UNNECESSARY=$(grep -c '^Function.* has unnecessary root' "$1"/unnecessary.txt) + NUM_DROPPED=$(grep -c '^Dropped CFG' "$1"/build_xgill.log) + NUM_WRITE_HAZARDS=$(perl -lne 'print $1 if m!found (\d+)/\d+ allowed errors!' "$1"/heapWriteHazards.txt) + NUM_MISSING=$(grep -c '^Function.*expected hazard.*but none were found' "$1"/rootingHazards.txt) + + set +x + echo "TinderboxPrint: rooting hazards<br/>$NUM_HAZARDS" + echo "TinderboxPrint: (unsafe references to unrooted GC pointers)<br/>$NUM_UNSAFE" + echo "TinderboxPrint: (unnecessary roots)<br/>$NUM_UNNECESSARY" + echo "TinderboxPrint: missing expected hazards<br/>$NUM_MISSING" + echo "TinderboxPrint: heap write hazards<br/>$NUM_WRITE_HAZARDS" + + # Display errors in a way that will get picked up by the taskcluster scraper. + perl -lne 'print "TEST-UNEXPECTED-FAIL | hazards | $1 $2" if /^Function.* has (unrooted .*live across GC call).* (at .*)$/' "$1"/hazards.txt + + exit_status=0 + + if [ $NUM_HAZARDS -gt 0 ]; then + echo "TEST-UNEXPECTED-FAIL | hazards | $NUM_HAZARDS rooting hazards detected" >&2 + echo "TinderboxPrint: documentation<br/><a href='https://wiki.mozilla.org/Javascript:Hazard_Builds#Diagnosing_a_rooting_hazards_failure'>static rooting hazard analysis failures</a>, visit \"Inspect Task\" link for hazard details" + exit_status=1 + fi + + if [ $NUM_MISSING -gt 0 ]; then + echo "TEST-UNEXPECTED-FAIL | hazards | $NUM_MISSING expected hazards went undetected" >&2 + echo "TinderboxPrint: documentation<br/><a href='https://wiki.mozilla.org/Javascript:Hazard_Builds#Diagnosing_a_rooting_hazards_failure'>static rooting hazard analysis failures</a>, visit \"Inspect Task\" link for hazard details" + exit_status=1 + fi + + NUM_ALLOWED_WRITE_HAZARDS=0 + if [ $NUM_WRITE_HAZARDS -gt $NUM_ALLOWED_WRITE_HAZARDS ]; then + echo "TEST-UNEXPECTED-FAIL | heap-write-hazards | $NUM_WRITE_HAZARDS heap write hazards detected out of $NUM_ALLOWED_WRITE_HAZARDS allowed" >&2 + echo "TinderboxPrint: documentation<br/><a href='https://wiki.mozilla.org/Javascript:Hazard_Builds#Diagnosing_a_heap_write_hazard_failure'>heap write hazard analysis failures</a>, visit \"Inspect Task\" link for hazard details" + exit_status = 1 + fi + + if [ $NUM_DROPPED -gt 0 ]; then + echo "TEST-UNEXPECTED-FAIL | hazards | $NUM_DROPPED CFGs dropped" >&2 + echo "TinderboxPrint: sixgill unable to handle constructs<br/>$NUM_DROPPED" + exit_status=1 + fi + + if [ $exit_status -ne 0 ]; then + exit $exit_status + fi + ) +} + +trap grab_artifacts EXIT + +# Directory to hold the (useless) object files generated by the analysis. +export HAZ_OBJDIR="$WORKSPACE/obj-analyzed-$PROJECT" +mkdir -p "$HAZ_OBJDIR" + +export LD_LIBRARY_PATH="$MOZ_FETCHES_DIR/gcc/lib64:$LD_LIBRARY_PATH" + +# Gather the information from the source tree by compiling it. +$GECKO_PATH/mach hazards gather --application=$PROJECT --haz-objdir="$HAZ_OBJDIR" --work-dir="$ANALYSIS_DIR" + +# Analyze the collected information. +$GECKO_PATH/mach hazards analyze --application=$PROJECT --shell-objdir="$HAZARD_SHELL_OBJDIR" --work-dir="$ANALYSIS_DIR" + +check_hazards "$ANALYSIS_DIR" + +################################### script end ################################### diff --git a/taskcluster/scripts/builder/build-l10n.sh b/taskcluster/scripts/builder/build-l10n.sh new file mode 100755 index 0000000000..e9e9db81e8 --- /dev/null +++ b/taskcluster/scripts/builder/build-l10n.sh @@ -0,0 +1,111 @@ +#! /bin/bash -vex + +set -x -e + +echo "running as" $(id) + +. /builds/worker/scripts/xvfb.sh + +#### +# Taskcluster friendly wrapper for performing fx desktop l10n repacks via mozharness. +# Based on ./build-linux.sh +#### + +# Inputs, with defaults + +: MOZHARNESS_SCRIPT ${MOZHARNESS_SCRIPT} +: MOZHARNESS_CONFIG ${MOZHARNESS_CONFIG} +: MOZHARNESS_CONFIG_PATHS ${MOZHARNESS_CONFIG_PATHS} +: MOZHARNESS_ACTIONS ${MOZHARNESS_ACTIONS} +: MOZHARNESS_OPTIONS ${MOZHARNESS_OPTIONS} + +: TOOLTOOL_CACHE ${TOOLTOOL_CACHE:=/builds/worker/tooltool-cache} + +: MOZ_SCM_LEVEL ${MOZ_SCM_LEVEL:=1} + +: NEED_XVFB ${NEED_XVFB:=false} + +: MOZ_SCM_LEVEL ${MOZ_SCM_LEVEL:=1} + +: WORKSPACE ${WORKSPACE:=/builds/worker/workspace} +: MOZ_OBJDIR ${MOZ_OBJDIR:=$WORKSPACE/obj-build} + +set -v + +fail() { + echo # make sure error message is on a new line + echo "[build-l10n.sh:error]" "${@}" + exit 1 +} + +export MOZ_CRASHREPORTER_NO_REPORT=1 +export TINDERBOX_OUTPUT=1 + +# Ensure that in tree libraries can be found +export LIBRARY_PATH=$LIBRARY_PATH:$WORKSPACE/obj-build:$WORKSPACE/src/gcc/lib64 + +# test required parameters are supplied +if [[ -z ${MOZHARNESS_SCRIPT} ]]; then fail "MOZHARNESS_SCRIPT is not set"; fi +if [[ -z "${MOZHARNESS_CONFIG}" && -z "${EXTRA_MOZHARNESS_CONFIG}" ]]; then fail "MOZHARNESS_CONFIG or EXTRA_MOZHARNESS_CONFIG is not set"; fi + +cleanup() { + local rv=$? + cleanup_xvfb + exit $rv +} +trap cleanup EXIT INT + +# run XVfb in the background, if necessary +if $NEED_XVFB; then + start_xvfb '1024x768x24' 2 +fi + +# set up mozharness configuration, via command line, env, etc. + +# $TOOLTOOL_CACHE bypasses mozharness completely and is read by tooltool_wrapper.sh to set the +# cache. However, only some mozharness scripts use tooltool_wrapper.sh, so this may not be +# entirely effective. +export TOOLTOOL_CACHE + +export MOZ_OBJDIR + +config_path_cmds="" +for path in ${MOZHARNESS_CONFIG_PATHS}; do + config_path_cmds="${config_path_cmds} --extra-config-path ${GECKO_PATH}/${path}" +done + +# support multiple, space delimited, config files +config_cmds="" +for cfg in $MOZHARNESS_CONFIG; do + config_cmds="${config_cmds} --config ${cfg}" +done + +# if MOZHARNESS_ACTIONS is given, only run those actions (completely overriding default_actions +# in the mozharness configuration) +if [ -n "$MOZHARNESS_ACTIONS" ]; then + actions="" + for action in $MOZHARNESS_ACTIONS; do + actions="$actions --$action" + done +fi + +# if MOZHARNESS_OPTIONS is given, append them to mozharness command line run +if [ -n "$MOZHARNESS_OPTIONS" ]; then + options="" + for option in $MOZHARNESS_OPTIONS; do + options="$options --$option" + done +fi + +cd /builds/worker + +$GECKO_PATH/mach python \ + --requirements $GECKO_PATH/taskcluster/scripts/builder/requirements.txt \ + -- \ + $GECKO_PATH/testing/${MOZHARNESS_SCRIPT} \ + ${config_path_cmds} \ + ${config_cmds} \ + $actions \ + $options \ + --log-level=debug \ + --work-dir=$WORKSPACE \ diff --git a/taskcluster/scripts/builder/build-linux.sh b/taskcluster/scripts/builder/build-linux.sh new file mode 100755 index 0000000000..bcc5b34606 --- /dev/null +++ b/taskcluster/scripts/builder/build-linux.sh @@ -0,0 +1,130 @@ +#! /bin/bash -vex + +set -x -e + +echo "running as" $(id) + +#### +# Taskcluster friendly wrapper for performing fx desktop builds via mozharness. +#### + +# Inputs, with defaults + +: MOZHARNESS_SCRIPT ${MOZHARNESS_SCRIPT} +: MOZHARNESS_CONFIG ${MOZHARNESS_CONFIG} +: MOZHARNESS_CONFIG_PATHS ${MOZHARNESS_CONFIG_PATHS} +: MOZHARNESS_ACTIONS ${MOZHARNESS_ACTIONS} +: MOZHARNESS_OPTIONS ${MOZHARNESS_OPTIONS} + +: TOOLTOOL_CACHE ${TOOLTOOL_CACHE:=/builds/worker/tooltool-cache} + +: MOZ_SCM_LEVEL ${MOZ_SCM_LEVEL:=1} + +: NEED_XVFB ${NEED_XVFB:=false} + +: MH_CUSTOM_BUILD_VARIANT_CFG ${MH_CUSTOM_BUILD_VARIANT_CFG} +: MH_BRANCH ${MH_BRANCH:=mozilla-central} +: MH_BUILD_POOL ${MH_BUILD_POOL:=staging} + +: WORKSPACE ${WORKSPACE:=/builds/worker/workspace} +: MOZ_OBJDIR ${MOZ_OBJDIR:=$WORKSPACE/obj-build} + +set -v + +fail() { + echo # make sure error message is on a new line + echo "[build-linux.sh:error]" "${@}" + exit 1 +} + +export MOZ_CRASHREPORTER_NO_REPORT=1 +export TINDERBOX_OUTPUT=1 + +# use "simple" package names so that they can be hard-coded in the task's +# extras.locations +export MOZ_SIMPLE_PACKAGE_NAME=target + +# Ensure that in tree libraries can be found +export LIBRARY_PATH=$LIBRARY_PATH:$WORKSPACE/obj-build:$WORKSPACE/src/gcc/lib64 + +# test required parameters are supplied +if [[ -z ${MOZHARNESS_SCRIPT} ]]; then fail "MOZHARNESS_SCRIPT is not set"; fi +if [[ -z "${MOZHARNESS_CONFIG}" && -z "${EXTRA_MOZHARNESS_CONFIG}" ]]; then fail "MOZHARNESS_CONFIG or EXTRA_MOZHARNESS_CONFIG is not set"; fi + +# run XVfb in the background, if necessary +if $NEED_XVFB; then + . /builds/worker/scripts/xvfb.sh + + cleanup() { + local rv=$? + cleanup_xvfb + exit $rv + } + trap cleanup EXIT INT + + start_xvfb '1024x768x24' 2 +fi + +# set up mozharness configuration, via command line, env, etc. + +debug_flag="" +if [ 0$DEBUG -ne 0 ]; then + debug_flag='--debug' +fi + +custom_build_variant_cfg_flag="" +if [ -n "${MH_CUSTOM_BUILD_VARIANT_CFG}" ]; then + custom_build_variant_cfg_flag="--custom-build-variant-cfg=${MH_CUSTOM_BUILD_VARIANT_CFG}" +fi + +# $TOOLTOOL_CACHE bypasses mozharness completely and is read by tooltool_wrapper.sh to set the +# cache. However, only some mozharness scripts use tooltool_wrapper.sh, so this may not be +# entirely effective. +export TOOLTOOL_CACHE + +export MOZ_OBJDIR + +config_path_cmds="" +for path in ${MOZHARNESS_CONFIG_PATHS}; do + config_path_cmds="${config_path_cmds} --extra-config-path ${GECKO_PATH}/${path}" +done + +# support multiple, space delimited, config files +config_cmds="" +for cfg in $MOZHARNESS_CONFIG; do + config_cmds="${config_cmds} --config ${cfg}" +done + +# if MOZHARNESS_ACTIONS is given, only run those actions (completely overriding default_actions +# in the mozharness configuration) +if [ -n "$MOZHARNESS_ACTIONS" ]; then + actions="" + for action in $MOZHARNESS_ACTIONS; do + actions="$actions --$action" + done +fi + +# if MOZHARNESS_OPTIONS is given, append them to mozharness command line run +if [ -n "$MOZHARNESS_OPTIONS" ]; then + options="" + for option in $MOZHARNESS_OPTIONS; do + options="$options --$option" + done +fi + +cd /builds/worker + +$GECKO_PATH/mach python \ + --requirements $GECKO_PATH/taskcluster/scripts/builder/requirements.txt \ + -- \ + $GECKO_PATH/testing/${MOZHARNESS_SCRIPT} \ + ${config_path_cmds} \ + ${config_cmds} \ + $debug_flag \ + $custom_build_variant_cfg_flag \ + $actions \ + $options \ + --log-level=debug \ + --work-dir=$WORKSPACE \ + --branch=${MH_BRANCH} \ + --build-pool=${MH_BUILD_POOL} diff --git a/taskcluster/scripts/builder/build-sm-mozjs-crate.sh b/taskcluster/scripts/builder/build-sm-mozjs-crate.sh new file mode 100755 index 0000000000..1d16de037a --- /dev/null +++ b/taskcluster/scripts/builder/build-sm-mozjs-crate.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -xe + +source $(dirname $0)/sm-tooltool-config.sh + +cd "$GECKO_PATH/js/src" + +cp $GECKO_PATH/.cargo/config.in $GECKO_PATH/.cargo/config + +export PATH="$PATH:$MOZ_FETCHES_DIR/cargo/bin:$MOZ_FETCHES_DIR/rustc/bin" +export RUSTFMT="$MOZ_FETCHES_DIR/rustc/bin/rustfmt" +export RUST_BACKTRACE=1 +export AUTOMATION=1 + +cargo build --verbose --frozen --features debugmozjs +cargo build --verbose --frozen diff --git a/taskcluster/scripts/builder/build-sm-package.sh b/taskcluster/scripts/builder/build-sm-package.sh new file mode 100755 index 0000000000..ec8147cb34 --- /dev/null +++ b/taskcluster/scripts/builder/build-sm-package.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +set -xe + +source $(dirname $0)/sm-tooltool-config.sh + +mkdir -p $UPLOAD_DIR + +# Package up the sources into the release tarball. +AUTOMATION=1 DIST=$UPLOAD_DIR $GECKO_PATH/js/src/make-source-package.sh + +# Extract the tarball into a new directory in the workspace. + +PACKAGE_DIR=$WORK/sm-package + +# Do not use -p option because the package directory should not exist. +mkdir $PACKAGE_DIR +pushd $PACKAGE_DIR + +tar -xvf $UPLOAD_DIR/mozjs-*.tar.*z* + +: ${PYTHON3:=python3} + +status=0 +( + # Build the freshly extracted, packaged SpiderMonkey. + cd ./mozjs-*/js/src + + # MOZ_AUTOMATION enforces certain requirements that don't apply to + # packaged builds. Unset it. + unset MOZ_AUTOMATION + + AUTOMATION=1 $PYTHON3 ./devtools/automation/autospider.py --skip-tests=checks $SPIDERMONKEY_VARIANT +) || status=$? + +# Copy artifacts for upload by TaskCluster +cp -rL ./mozjs-*/obj-spider/dist/bin/{js,jsapi-tests,js-gdb.py,libmozjs*} $UPLOAD_DIR + +exit $status diff --git a/taskcluster/scripts/builder/build-sm-rust-bindings.sh b/taskcluster/scripts/builder/build-sm-rust-bindings.sh new file mode 100755 index 0000000000..e749712e70 --- /dev/null +++ b/taskcluster/scripts/builder/build-sm-rust-bindings.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash + +set -xe + +source $(dirname $0)/sm-tooltool-config.sh + +cd "$GECKO_PATH/js/rust" + +cp $GECKO_PATH/.cargo/config.in $GECKO_PATH/.cargo/config + +export RUSTFMT="$MOZ_FETCHES_DIR/rustc/bin/rustfmt" +export LD_LIBRARY_PATH="$MOZ_FETCHES_DIR/gcc/lib64" +# Enable backtraces if we panic. +export RUST_BACKTRACE=1 + +cargo test --verbose --frozen --features debugmozjs +cargo test --verbose --frozen diff --git a/taskcluster/scripts/builder/build-sm.sh b/taskcluster/scripts/builder/build-sm.sh new file mode 100755 index 0000000000..70cb163077 --- /dev/null +++ b/taskcluster/scripts/builder/build-sm.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +set -x + +source $(dirname $0)/sm-tooltool-config.sh + +: ${PYTHON3:=python3} + +# Ensure upload dir exists +mkdir -p $UPLOAD_DIR + +# Run the script +export MOZ_UPLOAD_DIR="$(cd "$UPLOAD_DIR"; pwd)" +export OBJDIR=$WORK/obj-spider +AUTOMATION=1 $PYTHON3 $GECKO_PATH/js/src/devtools/automation/autospider.py ${SPIDERMONKEY_PLATFORM:+--platform=$SPIDERMONKEY_PLATFORM} $SPIDERMONKEY_VARIANT +BUILD_STATUS=$? + +# Copy artifacts for upload by TaskCluster. +upload=${MOZ_AUTOMATION_UPLOAD-1} +# User-provided override switch. +if [ -n "$MOZ_JS_UPLOAD_BINARIES" ]; then + upload=1 +fi +if [ "$upload" = "1" ]; then + cp -rL $OBJDIR/dist/bin/{js,jsapi-tests,js-gdb.py} $UPLOAD_DIR + + # Fuzzing users want the correct version of llvm-symbolizer available in the + # same directory as the built output. + for f in "$MOZ_FETCHES_DIR/clang/bin/llvm-symbolizer"*; do + gzip -c "$f" > "$UPLOAD_DIR/llvm-symbolizer.gz" || echo "gzip $f failed" >&2 + break + done + + # Fuzzing also uses a few fields in target.json file for automated downloads to + # identify what was built. + if [ -n "$MOZ_BUILD_DATE" ] && [ -n "$GECKO_HEAD_REV" ]; then + cat >$UPLOAD_DIR/target.json <<EOF +{ + "buildid": "$MOZ_BUILD_DATE", + "moz_source_stamp": "$GECKO_HEAD_REV" +} +EOF + fi +else # !upload + +# Provide a note for users on why we don't include artifacts for these builds +# by default, and how they can get the artifacts if they really need them. +cat >$UPLOAD_DIR/README-artifacts.txt <<'EOF' +Artifact upload has been disabled for this build due to infrequent usage of the +generated artifacts. If you find yourself in a position where you need the +shell or similar artifacts from this build, please redo your push with the +environment variable MOZ_JS_UPLOAD_BINARIES set to 1. You can provide this as +the option `--env MOZ_JS_UPLOAD_BINARIES=1` to `mach try fuzzy` or `mach try auto`. +EOF +fi + +exit $BUILD_STATUS diff --git a/taskcluster/scripts/builder/repackage.sh b/taskcluster/scripts/builder/repackage.sh new file mode 100755 index 0000000000..15ddbe8557 --- /dev/null +++ b/taskcluster/scripts/builder/repackage.sh @@ -0,0 +1,104 @@ +#! /bin/bash -vex + +set -x -e + +echo "running as" $(id) + +. /builds/worker/scripts/xvfb.sh + +#### +# Taskcluster friendly wrapper for performing fx desktop builds via mozharness. +#### + +# Inputs, with defaults + +: MOZHARNESS_SCRIPT ${MOZHARNESS_SCRIPT} +: MOZHARNESS_CONFIG ${MOZHARNESS_CONFIG} +: MOZHARNESS_CONFIG_PATHS ${MOZHARNESS_CONFIG_PATHS} +: MOZHARNESS_ACTIONS ${MOZHARNESS_ACTIONS} +: MOZHARNESS_OPTIONS ${MOZHARNESS_OPTIONS} + +: TOOLTOOL_CACHE ${TOOLTOOL_CACHE:=/builds/worker/tooltool-cache} + +: MOZ_SCM_LEVEL ${MOZ_SCM_LEVEL:=1} + +: WORKSPACE ${WORKSPACE:=/builds/worker/workspace} +: MOZ_OBJDIR ${MOZ_OBJDIR:=$WORKSPACE/obj-build} + +set -v + +fail() { + echo # make sure error message is on a new line + echo "[build-linux.sh:error]" "${@}" + exit 1 +} + +export MOZ_CRASHREPORTER_NO_REPORT=1 +export TINDERBOX_OUTPUT=1 + +# use "simple" package names so that they can be hard-coded in the task's +# extras.locations +export MOZ_SIMPLE_PACKAGE_NAME=target + +# test required parameters are supplied +if [[ -z ${MOZHARNESS_SCRIPT} ]]; then fail "MOZHARNESS_SCRIPT is not set"; fi +if [[ -z "${MOZHARNESS_CONFIG}" && -z "${EXTRA_MOZHARNESS_CONFIG}" ]]; then fail "MOZHARNESS_CONFIG or EXTRA_MOZHARNESS_CONFIG is not set"; fi + +cleanup() { + local rv=$? + cleanup_xvfb + exit $rv +} +trap cleanup EXIT INT + +# set up mozharness configuration, via command line, env, etc. + +debug_flag="" +if [ 0$DEBUG -ne 0 ]; then + debug_flag='--debug' +fi + +# $TOOLTOOL_CACHE bypasses mozharness completely and is read by tooltool_wrapper.sh to set the +# cache. However, only some mozharness scripts use tooltool_wrapper.sh, so this may not be +# entirely effective. +export TOOLTOOL_CACHE + +export MOZ_OBJDIR + +config_path_cmds="" +for path in ${MOZHARNESS_CONFIG_PATHS}; do + config_path_cmds="${config_path_cmds} --extra-config-path ${GECKO_PATH}/${path}" +done + +# support multiple, space delimited, config files +config_cmds="" +for cfg in $MOZHARNESS_CONFIG; do + config_cmds="${config_cmds} --config ${cfg}" +done + +# if MOZHARNESS_ACTIONS is given, only run those actions (completely overriding default_actions +# in the mozharness configuration) +if [ -n "$MOZHARNESS_ACTIONS" ]; then + actions="" + for action in $MOZHARNESS_ACTIONS; do + actions="$actions --$action" + done +fi + +# if MOZHARNESS_OPTIONS is given, append them to mozharness command line run +if [ -n "$MOZHARNESS_OPTIONS" ]; then + options="" + for option in $MOZHARNESS_OPTIONS; do + options="$options --$option" + done +fi + +cd /builds/worker + +$GECKO_PATH/mach python $GECKO_PATH/testing/${MOZHARNESS_SCRIPT} \ + ${config_path_cmds} \ + ${config_cmds} \ + $actions \ + $options \ + --log-level=debug \ + --work-dir=$WORKSPACE \ diff --git a/taskcluster/scripts/builder/requirements.txt b/taskcluster/scripts/builder/requirements.txt new file mode 100644 index 0000000000..24156d6d30 --- /dev/null +++ b/taskcluster/scripts/builder/requirements.txt @@ -0,0 +1 @@ +psutil==5.7.0 diff --git a/taskcluster/scripts/builder/sm-tooltool-config.sh b/taskcluster/scripts/builder/sm-tooltool-config.sh new file mode 100755 index 0000000000..380fd2e131 --- /dev/null +++ b/taskcluster/scripts/builder/sm-tooltool-config.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +set -x + +SPIDERMONKEY_VARIANT=${SPIDERMONKEY_VARIANT:-plain} +UPLOAD_DIR=${UPLOAD_DIR:-$HOME/artifacts/} +WORK=${WORK:-$HOME/workspace} + +export TOOLTOOL_CHECKOUT=${TOOLTOOL_CHECKOUT:-$WORK} + +( # Create scope for set -e +set -e +mkdir -p $WORK +cd $WORK + +# Need to install things from tooltool. Figure out what platform to use. + +case $(uname -m) in + i686 | arm ) + BITS=32 + ;; + *) + BITS=64 + ;; +esac + +case "$OSTYPE" in + darwin*) + PLATFORM_OS=macosx + ;; + linux-gnu) + PLATFORM_OS=linux + ;; + msys) + PLATFORM_OS=win + ;; + *) + echo "Unrecognized OSTYPE '$OSTYPE'" >&2 + PLATFORM_OS=linux + ;; +esac + +# Install everything needed for the browser on this platform. Not all of it is +# necessary for the JS shell, but it's less duplication to share tooltool +# manifests. +BROWSER_PLATFORM=$PLATFORM_OS$BITS + +(cd $TOOLTOOL_CHECKOUT && ${GECKO_PATH}/mach artifact toolchain${TOOLTOOL_MANIFEST:+ -v --tooltool-manifest $GECKO_PATH/$TOOLTOOL_MANIFEST}${TOOLTOOL_CACHE:+ --cache-dir $TOOLTOOL_CACHE}) + +) || exit 1 # end of set -e scope + +# Add all the fetches and tooltool binaries to our $PATH. +for bin in $MOZ_FETCHES_DIR/*/bin $TOOLTOOL_CHECKOUT/VC/bin/Hostx64/x86; do + if [ ! -d "$bin" ]; then + continue + fi + absbin=$(cd "$bin" && pwd) + export PATH="$absbin:$PATH" +done + +if [ -e $MOZ_FETCHES_DIR/rustc ]; then + export RUSTC="$MOZ_FETCHES_DIR/rustc/bin/rustc" + export CARGO="$MOZ_FETCHES_DIR/rustc/bin/cargo" +fi +if [ -e $MOZ_FETCHES_DIR/cbindgen ]; then + export CBINDGEN="$MOZ_FETCHES_DIR/cbindgen/cbindgen" +fi |