diff options
Diffstat (limited to 'src/rocksdb/build_tools/rocksdb-lego-determinator')
| -rwxr-xr-x | src/rocksdb/build_tools/rocksdb-lego-determinator | 1068 |
1 files changed, 1068 insertions, 0 deletions
diff --git a/src/rocksdb/build_tools/rocksdb-lego-determinator b/src/rocksdb/build_tools/rocksdb-lego-determinator new file mode 100755 index 000000000..948488c49 --- /dev/null +++ b/src/rocksdb/build_tools/rocksdb-lego-determinator @@ -0,0 +1,1068 @@ +#!/usr/bin/env bash +# This script is executed by Sandcastle +# to determine next steps to run + +# Usage: +# EMAIL=<email> ONCALL=<email> TRIGGER=<trigger> SUBSCRIBER=<email> rocks_ci.py <test-name> +# +# Input Value +# ------------------------------------------------------------------------- +# EMAIL Email address to report on trigger conditions +# ONCALL Email address to raise a task on failure +# TRIGGER Trigger conditions for email. Valid values are fail, warn, all +# SUBSCRIBER Email addresss to add as subscriber for task +# + +# +# Report configuration +# +REPORT_EMAIL= +if [ ! -z $EMAIL ]; then + if [ -z $TRIGGER ]; then + TRIGGER="fail" + fi + + REPORT_EMAIL=" + { + 'type':'email', + 'triggers': [ '$TRIGGER' ], + 'emails':['$EMAIL'] + }," +fi + +CREATE_TASK= +if [ ! -z $ONCALL ]; then + CREATE_TASK=" + { + 'type':'task', + 'triggers':[ 'fail' ], + 'priority':0, + 'subscribers':[ '$SUBSCRIBER' ], + 'tags':[ 'rocksdb', 'ci' ], + }," +fi + +# For now, create the tasks using only the dedicated task creation tool. +CREATE_TASK= + +REPORT= +if [[ ! -z $REPORT_EMAIL || ! -z $CREATE_TASK ]]; then + REPORT="'report': [ + $REPORT_EMAIL + $CREATE_TASK + ]" +fi + +# +# Helper variables +# +CLEANUP_ENV=" +{ + 'name':'Cleanup environment', + 'shell':'rm -rf /dev/shm/rocksdb && mkdir /dev/shm/rocksdb && (chmod +t /dev/shm || true) && make clean', + 'user':'root' +}" + +UPLOAD_DB_DIR=" +{ + 'name':'Upload database directory', + 'shell':'tar -cvzf rocksdb_db.tar.gz /dev/shm/rocksdb/', + 'user':'root', + 'cleanup':true, + 'provide_artifacts': [ + { + 'name':'rocksdb_db_dir', + 'paths': ['rocksdb_db.tar.gz'], + 'bundle': false, + }, + ], +}" + +# We will eventually set the RATIO to 1, but we want do this +# in steps. RATIO=$(nproc) will make it work as J=1 +if [ -z $RATIO ]; then + RATIO=$(nproc) +fi + +if [ -z $PARALLEL_J ]; then + PARALLEL_J="J=$(expr $(nproc) / ${RATIO})" +fi + +if [ -z $PARALLEL_j ]; then + PARALLEL_j="-j$(expr $(nproc) / ${RATIO})" +fi + +PARALLELISM="$PARALLEL_J $PARALLEL_j" + +DEBUG="OPT=-g" +SHM="TEST_TMPDIR=/dev/shm/rocksdb" +NON_SHM="TMPD=/tmp/rocksdb_test_tmp" +GCC_481="ROCKSDB_FBCODE_BUILD_WITH_481=1" +ASAN="COMPILE_WITH_ASAN=1" +CLANG="USE_CLANG=1" +# in gcc-5 there are known problems with TSAN like https://gcc.gnu.org/bugzilla/show_bug.cgi?id=71090. +# using platform007 gives us gcc-8 or higher which has that bug fixed. +TSAN="ROCKSDB_FBCODE_BUILD_WITH_PLATFORM007=1 COMPILE_WITH_TSAN=1" +UBSAN="COMPILE_WITH_UBSAN=1" +TSAN_CRASH='CRASH_TEST_EXT_ARGS="--compression_type=zstd --log2_keys_per_lock=22"' +NON_TSAN_CRASH="CRASH_TEST_EXT_ARGS=--compression_type=zstd" +DISABLE_JEMALLOC="DISABLE_JEMALLOC=1" +HTTP_PROXY="https_proxy=http://fwdproxy.29.prn1:8080 http_proxy=http://fwdproxy.29.prn1:8080 ftp_proxy=http://fwdproxy.29.prn1:8080" +SETUP_JAVA_ENV="export $HTTP_PROXY; export JAVA_HOME=/usr/local/jdk-8u60-64/; export PATH=\$JAVA_HOME/bin:\$PATH" +PARSER="'parser':'python build_tools/error_filter.py $1'" + +CONTRUN_NAME="ROCKSDB_CONTRUN_NAME" + +# This code is getting called under various scenarios. What we care about is to +# understand when it's called from nightly contruns because in that case we'll +# create tasks for any failures. To follow the existing pattern, we'll check +# the value of $ONCALL. If it's a diff then just call `false` to make sure +# that errors will be properly propagated to the caller. +if [ ! -z $ONCALL ]; then + TASK_CREATION_TOOL="/usr/local/bin/mysql_mtr_filter --rocksdb --oncall $ONCALL" +else + TASK_CREATION_TOOL="false" +fi + +# +# A mechanism to disable tests temporarily +# +DISABLE_COMMANDS="[ + { + 'name':'Disable test', + 'oncall':'$ONCALL', + 'steps': [ + { + 'name':'Job disabled. Please contact test owner', + 'shell':'exit 1', + 'user':'root' + }, + ], + } +]" + +# +# RocksDB unit test +# +UNIT_TEST_COMMANDS="[ + { + 'name':'Rocksdb Unit Test', + 'oncall':'$ONCALL', + 'executeLocal': 'true', + 'steps': [ + $CLEANUP_ENV, + { + 'name':'Build and test RocksDB debug version', + 'shell':'$SHM $DEBUG make $PARALLELISM check || $CONTRUN_NAME=check $TASK_CREATION_TOOL', + 'user':'root', + $PARSER + }, + ], + $REPORT + } +]" + +# +# RocksDB unit test not under /dev/shm +# +UNIT_TEST_NON_SHM_COMMANDS="[ + { + 'name':'Rocksdb Unit Test', + 'oncall':'$ONCALL', + 'executeLocal': 'true', + 'timeout': 86400, + 'steps': [ + $CLEANUP_ENV, + { + 'name':'Build and test RocksDB debug version', + 'timeout': 86400, + 'shell':'$NON_SHM $DEBUG make $PARALLELISM check || $CONTRUN_NAME=non_shm_check $TASK_CREATION_TOOL', + 'user':'root', + $PARSER + }, + ], + $REPORT + } +]" + +# +# RocksDB release build and unit tests +# +RELEASE_BUILD_COMMANDS="[ + { + 'name':'Rocksdb Release Build', + 'oncall':'$ONCALL', + 'executeLocal': 'true', + 'steps': [ + $CLEANUP_ENV, + { + 'name':'Build RocksDB release', + 'shell':'make $PARALLEL_j release || $CONTRUN_NAME=release $TASK_CREATION_TOOL', + 'user':'root', + $PARSER + }, + ], + $REPORT + } +]" + +# +# RocksDB unit test on gcc-4.8.1 +# +UNIT_TEST_COMMANDS_481="[ + { + 'name':'Rocksdb Unit Test on GCC 4.8.1', + 'oncall':'$ONCALL', + 'executeLocal': 'true', + 'steps': [ + $CLEANUP_ENV, + { + 'name':'Build and test RocksDB debug version', + 'shell':'$SHM $GCC_481 $DEBUG make $PARALLELISM check || $CONTRUN_NAME=unit_gcc_481_check $TASK_CREATION_TOOL', + 'user':'root', + $PARSER + }, + ], + $REPORT + } +]" + +# +# RocksDB release build and unit tests +# +RELEASE_BUILD_COMMANDS_481="[ + { + 'name':'Rocksdb Release on GCC 4.8.1', + 'oncall':'$ONCALL', + 'executeLocal': 'true', + 'steps': [ + $CLEANUP_ENV, + { + 'name':'Build RocksDB release on GCC 4.8.1', + 'shell':'$GCC_481 make $PARALLEL_j release || $CONTRUN_NAME=release_gcc481 $TASK_CREATION_TOOL', + 'user':'root', + $PARSER + }, + ], + $REPORT + } +]" + +# +# RocksDB unit test with CLANG +# +CLANG_UNIT_TEST_COMMANDS="[ + { + 'name':'Rocksdb Unit Test', + 'oncall':'$ONCALL', + 'executeLocal': 'true', + 'steps': [ + $CLEANUP_ENV, + { + 'name':'Build and test RocksDB debug', + 'shell':'$CLANG $SHM $DEBUG make $PARALLELISM check || $CONTRUN_NAME=clang_check $TASK_CREATION_TOOL', + 'user':'root', + $PARSER + }, + ], + $REPORT + } +]" + +# +# RocksDB release build with CLANG +# +CLANG_RELEASE_BUILD_COMMANDS="[ + { + 'name':'Rocksdb CLANG Release Build', + 'oncall':'$ONCALL', + 'executeLocal': 'true', + 'steps': [ + $CLEANUP_ENV, + { + 'name':'Build RocksDB release', + 'shell':'$CLANG make $PARALLEL_j release|| $CONTRUN_NAME=clang_release $TASK_CREATION_TOOL', + 'user':'root', + $PARSER + }, + ], + $REPORT + } +]" + +# +# RocksDB analyze +# +CLANG_ANALYZE_COMMANDS="[ + { + 'name':'Rocksdb analyze', + 'oncall':'$ONCALL', + 'executeLocal': 'true', + 'steps': [ + $CLEANUP_ENV, + { + 'name':'RocksDB build and analyze', + 'shell':'$CLANG $SHM $DEBUG make $PARALLEL_j analyze || $CONTRUN_NAME=clang_analyze $TASK_CREATION_TOOL', + 'user':'root', + $PARSER + }, + ], + $REPORT + } +]" + +# +# RocksDB code coverage +# +CODE_COV_COMMANDS="[ + { + 'name':'Rocksdb Unit Test Code Coverage', + 'oncall':'$ONCALL', + 'executeLocal': 'true', + 'steps': [ + $CLEANUP_ENV, + { + 'name':'Build, test and collect code coverage info', + 'shell':'$SHM $DEBUG make $PARALLELISM coverage || $CONTRUN_NAME=coverage $TASK_CREATION_TOOL', + 'user':'root', + $PARSER + }, + ], + $REPORT + } +]" + +# +# RocksDB unity +# +UNITY_COMMANDS="[ + { + 'name':'Rocksdb Unity', + 'oncall':'$ONCALL', + 'executeLocal': 'true', + 'steps': [ + $CLEANUP_ENV, + { + 'name':'Build, test unity test', + 'shell':'$SHM $DEBUG V=1 make J=1 unity_test || $CONTRUN_NAME=unity_test $TASK_CREATION_TOOL', + 'user':'root', + $PARSER + }, + ], + $REPORT + } +]" + +# +# Build RocksDB lite +# +LITE_BUILD_COMMANDS="[ + { + 'name':'Rocksdb Lite build', + 'oncall':'$ONCALL', + 'executeLocal': 'true', + 'steps': [ + $CLEANUP_ENV, + { + 'name':'Build RocksDB debug version', + 'shell':'make J=1 LITE=1 all check || $CONTRUN_NAME=lite $TASK_CREATION_TOOL', + 'user':'root', + $PARSER + }, + ], + $REPORT + } +]" + +# +# Report RocksDB lite binary size to scuba +REPORT_LITE_BINARY_SIZE_COMMANDS="[ + { + 'name':'Rocksdb Lite Binary Size', + 'oncall':'$ONCALL', + 'executeLocal': 'true', + 'steps': [ + $CLEANUP_ENV, + { + 'name':'Report RocksDB Lite binary size to scuba', + 'shell':'tools/report_lite_binary_size.sh', + 'user':'root', + }, + ], +]" + +# +# RocksDB stress/crash test +# +STRESS_CRASH_TEST_COMMANDS="[ + { + 'name':'Rocksdb Stress and Crash Test', + 'oncall':'$ONCALL', + 'executeLocal': 'true', + 'timeout': 86400, + 'steps': [ + $CLEANUP_ENV, + { + 'name':'Build and run RocksDB debug stress tests', + 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL', + 'user':'root', + $PARSER + }, + { + 'name':'Build and run RocksDB debug crash tests', + 'timeout': 86400, + 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 crash_test || $CONTRUN_NAME=crash_test $TASK_CREATION_TOOL', + 'user':'root', + $PARSER + }, + $UPLOAD_DB_DIR, + ], + $REPORT + } +]" + +# +# RocksDB stress/crash test with atomic flush +# +STRESS_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[ + { + 'name':'Rocksdb Stress and Crash Test with atomic flush', + 'oncall':'$ONCALL', + 'executeLocal': 'true', + 'timeout': 86400, + 'steps': [ + $CLEANUP_ENV, + { + 'name':'Build and run RocksDB debug stress tests', + 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL', + 'user':'root', + $PARSER + }, + { + 'name':'Build and run RocksDB debug crash tests with atomic flush', + 'timeout': 86400, + 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 crash_test_with_atomic_flush || $CONTRUN_NAME=crash_test_with_atomic_flush $TASK_CREATION_TOOL', + 'user':'root', + $PARSER + }, + $UPLOAD_DB_DIR, + ], + $REPORT + } +]" + +# +# RocksDB stress/crash test with txn +# +STRESS_CRASH_TEST_WITH_TXN_COMMANDS="[ + { + 'name':'Rocksdb Stress and Crash Test with txn', + 'oncall':'$ONCALL', + 'executeLocal': 'true', + 'timeout': 86400, + 'steps': [ + $CLEANUP_ENV, + { + 'name':'Build and run RocksDB debug stress tests', + 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 db_stress || $CONTRUN_NAME=db_stress $TASK_CREATION_TOOL', + 'user':'root', + $PARSER + }, + { + 'name':'Build and run RocksDB debug crash tests with txn', + 'timeout': 86400, + 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 crash_test_with_txn || $CONTRUN_NAME=crash_test_with_txn $TASK_CREATION_TOOL', + 'user':'root', + $PARSER + }, + $UPLOAD_DB_DIR, + ], + $REPORT + } +]" + +# RocksDB write stress test. +# We run on disk device on purpose (i.e. no $SHM) +# because we want to add some randomness to fsync commands +WRITE_STRESS_COMMANDS="[ + { + 'name':'Rocksdb Write Stress Test', + 'oncall':'$ONCALL', + 'executeLocal': 'true', + 'steps': [ + $CLEANUP_ENV, + { + 'name':'Build and run RocksDB write stress tests', + 'shell':'make write_stress && python tools/write_stress_runner.py --runtime_sec=3600 --db=/tmp/rocksdb_write_stress || $CONTRUN_NAME=write_stress $TASK_CREATION_TOOL', + 'user':'root', + $PARSER + } + ], + 'artifacts': [{'name': 'database', 'paths': ['/tmp/rocksdb_write_stress']}], + $REPORT + } +]" + + +# +# RocksDB test under address sanitizer +# +ASAN_TEST_COMMANDS="[ + { + 'name':'Rocksdb Unit Test under ASAN', + 'oncall':'$ONCALL', + 'executeLocal': 'true', + 'steps': [ + $CLEANUP_ENV, + { + 'name':'Test RocksDB debug under ASAN', +'shell':'set -o pipefail && ($SHM $ASAN $DEBUG make $PARALLELISM asan_check || $CONTRUN_NAME=asan_check $TASK_CREATION_TOOL) |& /usr/facebook/ops/scripts/asan_symbolize.py -d', + 'user':'root', + $PARSER + } + ], + $REPORT + } +]" + +# +# RocksDB crash testing under address sanitizer +# +ASAN_CRASH_TEST_COMMANDS="[ + { + 'name':'Rocksdb crash test under ASAN', + 'oncall':'$ONCALL', + 'executeLocal': 'true', + 'timeout': 86400, + 'steps': [ + $CLEANUP_ENV, + { + 'name':'Build and run RocksDB debug asan_crash_test', + 'timeout': 86400, + 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 asan_crash_test || $CONTRUN_NAME=asan_crash_test $TASK_CREATION_TOOL', + 'user':'root', + $PARSER + }, + $UPLOAD_DB_DIR, + ], + $REPORT + } +]" + +# +# RocksDB crash testing with atomic flush under address sanitizer +# +ASAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[ + { + 'name':'Rocksdb crash test with atomic flush under ASAN', + 'oncall':'$ONCALL', + 'executeLocal': 'true', + 'timeout': 86400, + 'steps': [ + $CLEANUP_ENV, + { + 'name':'Build and run RocksDB debug asan_crash_test_with_atomic_flush', + 'timeout': 86400, + 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 asan_crash_test_with_atomic_flush || $CONTRUN_NAME=asan_crash_test_with_atomic_flush $TASK_CREATION_TOOL', + 'user':'root', + $PARSER + }, + $UPLOAD_DB_DIR, + ], + $REPORT + } +]" + +# +# RocksDB crash testing with txn under address sanitizer +# +ASAN_CRASH_TEST_WITH_TXN_COMMANDS="[ + { + 'name':'Rocksdb crash test with txn under ASAN', + 'oncall':'$ONCALL', + 'executeLocal': 'true', + 'timeout': 86400, + 'steps': [ + $CLEANUP_ENV, + { + 'name':'Build and run RocksDB debug asan_crash_test_with_txn', + 'timeout': 86400, + 'shell':'$SHM $DEBUG $NON_TSAN_CRASH make J=1 asan_crash_test_with_txn || $CONTRUN_NAME=asan_crash_test_with_txn $TASK_CREATION_TOOL', + 'user':'root', + $PARSER + }, + $UPLOAD_DB_DIR, + ], + $REPORT + } +]" + +# +# RocksDB test under undefined behavior sanitizer +# +UBSAN_TEST_COMMANDS="[ + { + 'name':'Rocksdb Unit Test under UBSAN', + 'oncall':'$ONCALL', + 'executeLocal': 'true', + 'steps': [ + $CLEANUP_ENV, + { + 'name':'Test RocksDB debug under UBSAN', + 'shell':'set -o pipefail && $SHM $UBSAN $CLANG $DEBUG make $PARALLELISM ubsan_check || $CONTRUN_NAME=ubsan_check $TASK_CREATION_TOOL', + 'user':'root', + $PARSER + } + ], + $REPORT + } +]" + +# +# RocksDB crash testing under udnefined behavior sanitizer +# +UBSAN_CRASH_TEST_COMMANDS="[ + { + 'name':'Rocksdb crash test under UBSAN', + 'oncall':'$ONCALL', + 'executeLocal': 'true', + 'timeout': 86400, + 'steps': [ + $CLEANUP_ENV, + { + 'name':'Build and run RocksDB debug ubsan_crash_test', + 'timeout': 86400, + 'shell':'$SHM $DEBUG $NON_TSAN_CRASH $CLANG make J=1 ubsan_crash_test || $CONTRUN_NAME=ubsan_crash_test $TASK_CREATION_TOOL', + 'user':'root', + $PARSER + }, + $UPLOAD_DB_DIR, + ], + $REPORT + } +]" + +# +# RocksDB crash testing with atomic flush under undefined behavior sanitizer +# +UBSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[ + { + 'name':'Rocksdb crash test with atomic flush under UBSAN', + 'oncall':'$ONCALL', + 'executeLocal': 'true', + 'timeout': 86400, + 'steps': [ + $CLEANUP_ENV, + { + 'name':'Build and run RocksDB debug ubsan_crash_test_with_atomic_flush', + 'timeout': 86400, + 'shell':'$SHM $DEBUG $NON_TSAN_CRASH $CLANG make J=1 ubsan_crash_test_with_atomic_flush || $CONTRUN_NAME=ubsan_crash_test_with_atomic_flush $TASK_CREATION_TOOL', + 'user':'root', + $PARSER + }, + $UPLOAD_DB_DIR, + ], + $REPORT + } +]" + +# +# RocksDB crash testing with txn under undefined behavior sanitizer +# +UBSAN_CRASH_TEST_WITH_TXN_COMMANDS="[ + { + 'name':'Rocksdb crash test with txn under UBSAN', + 'oncall':'$ONCALL', + 'executeLocal': 'true', + 'timeout': 86400, + 'steps': [ + $CLEANUP_ENV, + { + 'name':'Build and run RocksDB debug ubsan_crash_test_with_txn', + 'timeout': 86400, + 'shell':'$SHM $DEBUG $NON_TSAN_CRASH $CLANG make J=1 ubsan_crash_test_with_txn || $CONTRUN_NAME=ubsan_crash_test_with_txn $TASK_CREATION_TOOL', + 'user':'root', + $PARSER + }, + $UPLOAD_DB_DIR, + ], + $REPORT + } +]" + +# +# RocksDB unit test under valgrind +# +VALGRIND_TEST_COMMANDS="[ + { + 'name':'Rocksdb Unit Test under valgrind', + 'oncall':'$ONCALL', + 'executeLocal': 'true', + 'timeout': 86400, + 'steps': [ + $CLEANUP_ENV, + { + 'name':'Run RocksDB debug unit tests', + 'timeout': 86400, + 'shell':'$SHM $DEBUG make $PARALLELISM valgrind_test || $CONTRUN_NAME=valgrind_check $TASK_CREATION_TOOL', + 'user':'root', + $PARSER + }, + ], + $REPORT + } +]" + +# +# RocksDB test under TSAN +# +TSAN_UNIT_TEST_COMMANDS="[ + { + 'name':'Rocksdb Unit Test under TSAN', + 'oncall':'$ONCALL', + 'executeLocal': 'true', + 'timeout': 86400, + 'steps': [ + $CLEANUP_ENV, + { + 'name':'Run RocksDB debug unit test', + 'timeout': 86400, + 'shell':'set -o pipefail && $SHM $DEBUG $TSAN make $PARALLELISM check || $CONTRUN_NAME=tsan_check $TASK_CREATION_TOOL', + 'user':'root', + $PARSER + }, + ], + $REPORT + } +]" + +# +# RocksDB crash test under TSAN +# +TSAN_CRASH_TEST_COMMANDS="[ + { + 'name':'Rocksdb Crash Test under TSAN', + 'oncall':'$ONCALL', + 'executeLocal': 'true', + 'timeout': 86400, + 'steps': [ + $CLEANUP_ENV, + { + 'name':'Compile and run', + 'timeout': 86400, + 'shell':'set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make J=1 crash_test || $CONTRUN_NAME=tsan_crash_test $TASK_CREATION_TOOL', + 'user':'root', + $PARSER + }, + $UPLOAD_DB_DIR, + ], + $REPORT + } +]" + +# +# RocksDB crash test with atomic flush under TSAN +# +TSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS="[ + { + 'name':'Rocksdb Crash Test with atomic flush under TSAN', + 'oncall':'$ONCALL', + 'executeLocal': 'true', + 'timeout': 86400, + 'steps': [ + $CLEANUP_ENV, + { + 'name':'Compile and run', + 'timeout': 86400, + 'shell':'set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make J=1 crash_test_with_atomic_flush || $CONTRUN_NAME=tsan_crash_test_with_atomic_flush $TASK_CREATION_TOOL', + 'user':'root', + $PARSER + }, + $UPLOAD_DB_DIR, + ], + $REPORT + } +]" + +# +# RocksDB crash test with txn under TSAN +# +TSAN_CRASH_TEST_WITH_TXN_COMMANDS="[ + { + 'name':'Rocksdb Crash Test with txn under TSAN', + 'oncall':'$ONCALL', + 'executeLocal': 'true', + 'timeout': 86400, + 'steps': [ + $CLEANUP_ENV, + { + 'name':'Compile and run', + 'timeout': 86400, + 'shell':'set -o pipefail && $SHM $DEBUG $TSAN $TSAN_CRASH CRASH_TEST_KILL_ODD=1887 make J=1 crash_test_with_txn || $CONTRUN_NAME=tsan_crash_test_with_txn $TASK_CREATION_TOOL', + 'user':'root', + $PARSER + }, + $UPLOAD_DB_DIR, + ], + $REPORT + } +]" + +# +# RocksDB format compatible +# + +run_format_compatible() +{ + export TEST_TMPDIR=/dev/shm/rocksdb + rm -rf /dev/shm/rocksdb + mkdir /dev/shm/rocksdb + + tools/check_format_compatible.sh +} + +FORMAT_COMPATIBLE_COMMANDS="[ + { + 'name':'Rocksdb Format Compatible tests', + 'oncall':'$ONCALL', + 'executeLocal': 'true', + 'steps': [ + $CLEANUP_ENV, + { + 'name':'Run RocksDB debug unit test', + 'shell':'build_tools/rocksdb-lego-determinator run_format_compatible || $CONTRUN_NAME=run_format_compatible $TASK_CREATION_TOOL', + 'user':'root', + $PARSER + }, + ], + $REPORT + } +]" + +# +# RocksDB no compression +# +run_no_compression() +{ + export TEST_TMPDIR=/dev/shm/rocksdb + rm -rf /dev/shm/rocksdb + mkdir /dev/shm/rocksdb + make clean + cat build_tools/fbcode_config.sh | grep -iv dzstd | grep -iv dzlib | grep -iv dlz4 | grep -iv dsnappy | grep -iv dbzip2 > .tmp.fbcode_config.sh + mv .tmp.fbcode_config.sh build_tools/fbcode_config.sh + cat Makefile | grep -v tools/ldb_test.py > .tmp.Makefile + mv .tmp.Makefile Makefile + make $DEBUG J=1 check +} + +NO_COMPRESSION_COMMANDS="[ + { + 'name':'Rocksdb No Compression tests', + 'oncall':'$ONCALL', + 'executeLocal': 'true', + 'steps': [ + $CLEANUP_ENV, + { + 'name':'Run RocksDB debug unit test', + 'shell':'build_tools/rocksdb-lego-determinator run_no_compression || $CONTRUN_NAME=run_no_compression $TASK_CREATION_TOOL', + 'user':'root', + $PARSER + }, + ], + $REPORT + } +]" + +# +# RocksDB regression +# +run_regression() +{ + time -v bash -vx ./build_tools/regression_build_test.sh $(mktemp -d $WORKSPACE/leveldb.XXXX) $(mktemp leveldb_test_stats.XXXX) + + # ======= report size to ODS ======== + + # parameters: $1 -- key, $2 -- value + function send_size_to_ods { + curl --silent "https://www.intern.facebook.com/intern/agent/ods_set.php?entity=rocksdb_build&key=rocksdb.build_size.$1&value=$2" \ + --connect-timeout 60 + } + + # === normal build === + make clean + make -j$(nproc) static_lib + send_size_to_ods static_lib $(stat --printf="%s" librocksdb.a) + strip librocksdb.a + send_size_to_ods static_lib_stripped $(stat --printf="%s" librocksdb.a) + + make -j$(nproc) shared_lib + send_size_to_ods shared_lib $(stat --printf="%s" `readlink -f librocksdb.so`) + strip `readlink -f librocksdb.so` + send_size_to_ods shared_lib_stripped $(stat --printf="%s" `readlink -f librocksdb.so`) + + # === lite build === + make clean + make LITE=1 -j$(nproc) static_lib + send_size_to_ods static_lib_lite $(stat --printf="%s" librocksdb.a) + strip librocksdb.a + send_size_to_ods static_lib_lite_stripped $(stat --printf="%s" librocksdb.a) + + make LITE=1 -j$(nproc) shared_lib + send_size_to_ods shared_lib_lite $(stat --printf="%s" `readlink -f librocksdb.so`) + strip `readlink -f librocksdb.so` + send_size_to_ods shared_lib_lite_stripped $(stat --printf="%s" `readlink -f librocksdb.so`) +} + +REGRESSION_COMMANDS="[ + { + 'name':'Rocksdb regression commands', + 'oncall':'$ONCALL', + 'steps': [ + $CLEANUP_ENV, + { + 'name':'Make and run script', + 'shell':'build_tools/rocksdb-lego-determinator run_regression || $CONTRUN_NAME=run_regression $TASK_CREATION_TOOL', + 'user':'root', + $PARSER + }, + ], + $REPORT + } +]" + +# +# RocksDB Java build +# +JAVA_BUILD_TEST_COMMANDS="[ + { + 'name':'Rocksdb Java Build', + 'oncall':'$ONCALL', + 'executeLocal': 'true', + 'steps': [ + $CLEANUP_ENV, + { + 'name':'Build RocksDB for Java', + 'shell':'$SETUP_JAVA_ENV; $SHM make rocksdbjava || $CONTRUN_NAME=rocksdbjava $TASK_CREATION_TOOL', + 'user':'root', + $PARSER + }, + ], + $REPORT + } +]" + + +case $1 in + unit) + echo $UNIT_TEST_COMMANDS + ;; + unit_non_shm) + echo $UNIT_TEST_NON_SHM_COMMANDS + ;; + release) + echo $RELEASE_BUILD_COMMANDS + ;; + unit_481) + echo $UNIT_TEST_COMMANDS_481 + ;; + release_481) + echo $RELEASE_BUILD_COMMANDS_481 + ;; + clang_unit) + echo $CLANG_UNIT_TEST_COMMANDS + ;; + clang_release) + echo $CLANG_RELEASE_BUILD_COMMANDS + ;; + clang_analyze) + echo $CLANG_ANALYZE_COMMANDS + ;; + code_cov) + echo $CODE_COV_COMMANDS + ;; + unity) + echo $UNITY_COMMANDS + ;; + lite) + echo $LITE_BUILD_COMMANDS + ;; + report_lite_binary_size) + echo $REPORT_LITE_BINARY_SIZE_COMMANDS + ;; + stress_crash) + echo $STRESS_CRASH_TEST_COMMANDS + ;; + stress_crash_with_atomic_flush) + echo $STRESS_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS + ;; + stress_crash_with_txn) + echo $STRESS_CRASH_TEST_WITH_TXN_COMMANDS + ;; + write_stress) + echo $WRITE_STRESS_COMMANDS + ;; + asan) + echo $ASAN_TEST_COMMANDS + ;; + asan_crash) + echo $ASAN_CRASH_TEST_COMMANDS + ;; + asan_crash_with_atomic_flush) + echo $ASAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS + ;; + asan_crash_with_txn) + echo $ASAN_CRASH_TEST_WITH_TXN_COMMANDS + ;; + ubsan) + echo $UBSAN_TEST_COMMANDS + ;; + ubsan_crash) + echo $UBSAN_CRASH_TEST_COMMANDS + ;; + ubsan_crash_with_atomic_flush) + echo $UBSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS + ;; + ubsan_crash_with_txn) + echo $UBSAN_CRASH_TEST_WITH_TXN_COMMANDS + ;; + valgrind) + echo $VALGRIND_TEST_COMMANDS + ;; + tsan) + echo $TSAN_UNIT_TEST_COMMANDS + ;; + tsan_crash) + echo $TSAN_CRASH_TEST_COMMANDS + ;; + tsan_crash_with_atomic_flush) + echo $TSAN_CRASH_TEST_WITH_ATOMIC_FLUSH_COMMANDS + ;; + tsan_crash_with_txn) + echo $TSAN_CRASH_TEST_WITH_TXN_COMMANDS + ;; + format_compatible) + echo $FORMAT_COMPATIBLE_COMMANDS + ;; + run_format_compatible) + run_format_compatible + ;; + no_compression) + echo $NO_COMPRESSION_COMMANDS + ;; + run_no_compression) + run_no_compression + ;; + regression) + echo $REGRESSION_COMMANDS + ;; + run_regression) + run_regression + ;; + java_build) + echo $JAVA_BUILD_TEST_COMMANDS + ;; + *) + echo "Invalid determinator command" + exit 1 + ;; +esac |