diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-21 11:54:28 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-21 11:54:28 +0000 |
commit | e6918187568dbd01842d8d1d2c808ce16a894239 (patch) | |
tree | 64f88b554b444a49f656b6c656111a145cbbaa28 /src/vstart.sh | |
parent | Initial commit. (diff) | |
download | ceph-upstream/18.2.2.tar.xz ceph-upstream/18.2.2.zip |
Adding upstream version 18.2.2.upstream/18.2.2
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to '')
-rwxr-xr-x | src/vstart.sh | 1876 |
1 files changed, 1876 insertions, 0 deletions
diff --git a/src/vstart.sh b/src/vstart.sh new file mode 100755 index 000000000..92da54e87 --- /dev/null +++ b/src/vstart.sh @@ -0,0 +1,1876 @@ +#!/usr/bin/env bash +# -*- mode:sh; tab-width:4; sh-basic-offset:4; indent-tabs-mode:nil -*- +# vim: softtabstop=4 shiftwidth=4 expandtab + +# abort on failure +set -e + +quoted_print() { + for s in "$@"; do + if [[ "$s" =~ \ ]]; then + printf -- "'%s' " "$s" + else + printf -- "$s " + fi + done + printf '\n' +} + +debug() { + "$@" >&2 +} + +prunb() { + debug quoted_print "$@" '&' + PATH=$CEPH_BIN:$PATH "$@" & +} + +prun() { + debug quoted_print "$@" + PATH=$CEPH_BIN:$PATH "$@" +} + + +if [ -n "$VSTART_DEST" ]; then + SRC_PATH=`dirname $0` + SRC_PATH=`(cd $SRC_PATH; pwd)` + + CEPH_DIR=$SRC_PATH + CEPH_BIN=${CEPH_BIN:-${PWD}/bin} + CEPH_LIB=${CEPH_LIB:-${PWD}/lib} + + CEPH_CONF_PATH=$VSTART_DEST + CEPH_DEV_DIR=$VSTART_DEST/dev + CEPH_OUT_DIR=$VSTART_DEST/out + CEPH_ASOK_DIR=$VSTART_DEST/asok + CEPH_OUT_CLIENT_DIR=${CEPH_OUT_CLIENT_DIR:-$CEPH_OUT_DIR} +fi + +get_cmake_variable() { + local variable=$1 + grep "${variable}:" CMakeCache.txt | cut -d "=" -f 2 +} + +# for running out of the CMake build directory +if [ -e CMakeCache.txt ]; then + # Out of tree build, learn source location from CMakeCache.txt + CEPH_ROOT=$(get_cmake_variable ceph_SOURCE_DIR) + CEPH_BUILD_DIR=`pwd` + [ -z "$MGR_PYTHON_PATH" ] && MGR_PYTHON_PATH=$CEPH_ROOT/src/pybind/mgr +fi + +# use CEPH_BUILD_ROOT to vstart from a 'make install' +if [ -n "$CEPH_BUILD_ROOT" ]; then + [ -z "$CEPH_BIN" ] && CEPH_BIN=$CEPH_BUILD_ROOT/bin + [ -z "$CEPH_LIB" ] && CEPH_LIB=$CEPH_BUILD_ROOT/lib + [ -z "$CEPH_EXT_LIB" ] && CEPH_EXT_LIB=$CEPH_BUILD_ROOT/external/lib + [ -z "$EC_PATH" ] && EC_PATH=$CEPH_LIB/erasure-code + [ -z "$OBJCLASS_PATH" ] && OBJCLASS_PATH=$CEPH_LIB/rados-classes + # make install should install python extensions into PYTHONPATH +elif [ -n "$CEPH_ROOT" ]; then + [ -z "$CEPHFS_SHELL" ] && CEPHFS_SHELL=$CEPH_ROOT/src/tools/cephfs/shell/cephfs-shell + [ -z "$PYBIND" ] && PYBIND=$CEPH_ROOT/src/pybind + [ -z "$CEPH_BIN" ] && CEPH_BIN=$CEPH_BUILD_DIR/bin + [ -z "$CEPH_ADM" ] && CEPH_ADM=$CEPH_BIN/ceph + [ -z "$INIT_CEPH" ] && INIT_CEPH=$CEPH_BIN/init-ceph + [ -z "$CEPH_LIB" ] && CEPH_LIB=$CEPH_BUILD_DIR/lib + [ -z "$CEPH_EXT_LIB" ] && CEPH_EXT_LIB=$CEPH_BUILD_DIR/external/lib + [ -z "$OBJCLASS_PATH" ] && OBJCLASS_PATH=$CEPH_LIB + [ -z "$EC_PATH" ] && EC_PATH=$CEPH_LIB + [ -z "$CEPH_PYTHON_COMMON" ] && CEPH_PYTHON_COMMON=$CEPH_ROOT/src/python-common +fi + +if [ -z "${CEPH_VSTART_WRAPPER}" ]; then + PATH=$(pwd):$PATH +fi + +[ -z "$PYBIND" ] && PYBIND=./pybind + +[ -n "$CEPH_PYTHON_COMMON" ] && CEPH_PYTHON_COMMON="$CEPH_PYTHON_COMMON:" +CYTHON_PYTHONPATH="$CEPH_LIB/cython_modules/lib.3" +export PYTHONPATH=$PYBIND:$CYTHON_PYTHONPATH:$CEPH_PYTHON_COMMON$PYTHONPATH + +export LD_LIBRARY_PATH=$CEPH_LIB:$CEPH_EXT_LIB:$LD_LIBRARY_PATH +export DYLD_LIBRARY_PATH=$CEPH_LIB:$CEPH_EXT_LIB:$DYLD_LIBRARY_PATH +# Suppress logging for regular use that indicated that we are using a +# development version. vstart.sh is only used during testing and +# development +export CEPH_DEV=1 + +[ -z "$CEPH_NUM_MON" ] && CEPH_NUM_MON="$MON" +[ -z "$CEPH_NUM_OSD" ] && CEPH_NUM_OSD="$OSD" +[ -z "$CEPH_NUM_MDS" ] && CEPH_NUM_MDS="$MDS" +[ -z "$CEPH_NUM_MGR" ] && CEPH_NUM_MGR="$MGR" +[ -z "$CEPH_NUM_FS" ] && CEPH_NUM_FS="$FS" +[ -z "$CEPH_NUM_RGW" ] && CEPH_NUM_RGW="$RGW" +[ -z "$GANESHA_DAEMON_NUM" ] && GANESHA_DAEMON_NUM="$NFS" + +# if none of the CEPH_NUM_* number is specified, kill the existing +# cluster. +if [ -z "$CEPH_NUM_MON" -a \ + -z "$CEPH_NUM_OSD" -a \ + -z "$CEPH_NUM_MDS" -a \ + -z "$CEPH_NUM_MGR" -a \ + -z "$GANESHA_DAEMON_NUM" ]; then + kill_all=1 +else + kill_all=0 +fi + +[ -z "$CEPH_NUM_MON" ] && CEPH_NUM_MON=3 +[ -z "$CEPH_NUM_OSD" ] && CEPH_NUM_OSD=3 +[ -z "$CEPH_NUM_MDS" ] && CEPH_NUM_MDS=3 +[ -z "$CEPH_NUM_MGR" ] && CEPH_NUM_MGR=1 +[ -z "$CEPH_NUM_FS" ] && CEPH_NUM_FS=1 +[ -z "$CEPH_MAX_MDS" ] && CEPH_MAX_MDS=1 +[ -z "$CEPH_NUM_RGW" ] && CEPH_NUM_RGW=0 +[ -z "$GANESHA_DAEMON_NUM" ] && GANESHA_DAEMON_NUM=0 + +[ -z "$CEPH_DIR" ] && CEPH_DIR="$PWD" +[ -z "$CEPH_DEV_DIR" ] && CEPH_DEV_DIR="$CEPH_DIR/dev" +[ -z "$CEPH_OUT_DIR" ] && CEPH_OUT_DIR="$CEPH_DIR/out" +[ -z "$CEPH_ASOK_DIR" ] && CEPH_ASOK_DIR="$CEPH_DIR/asok" +[ -z "$CEPH_RGW_PORT" ] && CEPH_RGW_PORT=8000 +[ -z "$CEPH_CONF_PATH" ] && CEPH_CONF_PATH=$CEPH_DIR +CEPH_OUT_CLIENT_DIR=${CEPH_OUT_CLIENT_DIR:-$CEPH_OUT_DIR} + +if [ $CEPH_NUM_OSD -gt 3 ]; then + OSD_POOL_DEFAULT_SIZE=3 +else + OSD_POOL_DEFAULT_SIZE=$CEPH_NUM_OSD +fi + +extra_conf="" +new=0 +standby=0 +debug=0 +trace=0 +ip="" +nodaemon=0 +redirect=0 +smallmds=0 +short=0 +crimson=0 +ec=0 +cephadm=0 +parallel=true +restart=1 +hitset="" +overwrite_conf=0 +cephx=1 #turn cephx on by default +gssapi_authx=0 +cache="" +if [ `uname` = FreeBSD ]; then + objectstore="memstore" +else + objectstore="bluestore" +fi +ceph_osd=ceph-osd +rgw_frontend="beast" +rgw_compression="" +lockdep=${LOCKDEP:-1} +spdk_enabled=0 # disable SPDK by default +pmem_enabled=0 +zoned_enabled=0 +io_uring_enabled=0 +with_jaeger=0 + +with_mgr_dashboard=true +if [[ "$(get_cmake_variable WITH_MGR_DASHBOARD_FRONTEND)" != "ON" ]] || + [[ "$(get_cmake_variable WITH_RBD)" != "ON" ]]; then + debug echo "ceph-mgr dashboard not built - disabling." + with_mgr_dashboard=false +fi +with_mgr_restful=false + +kstore_path= +declare -a block_devs +declare -a secondary_block_devs +secondary_block_devs_type="SSD" + +VSTART_SEC="client.vstart.sh" + +MON_ADDR="" +DASH_URLS="" +RESTFUL_URLS="" + +conf_fn="$CEPH_CONF_PATH/ceph.conf" +keyring_fn="$CEPH_CONF_PATH/keyring" +monmap_fn="/tmp/ceph_monmap.$$" +inc_osd_num=0 + +msgr="21" + +read -r -d '' usage <<EOF || true +usage: $0 [option]... \nex: MON=3 OSD=1 MDS=1 MGR=1 RGW=1 NFS=1 $0 -n -d +options: + -d, --debug + -t, --trace + -s, --standby_mds: Generate standby-replay MDS for each active + -l, --localhost: use localhost instead of hostname + -i <ip>: bind to specific ip + -n, --new + --valgrind[_{osd,mds,mon,rgw}] 'toolname args...' + --nodaemon: use ceph-run as wrapper for mon/osd/mds + --redirect-output: only useful with nodaemon, directs output to log file + --smallmds: limit mds cache memory limit + -m ip:port specify monitor address + -k keep old configuration files (default) + -x enable cephx (on by default) + -X disable cephx + -g --gssapi enable Kerberos/GSSApi authentication + -G disable Kerberos/GSSApi authentication + --hitset <pool> <hit_set_type>: enable hitset tracking + -e : create an erasure pool + -o config add extra config parameters to all sections + --rgw_port specify ceph rgw http listen port + --rgw_frontend specify the rgw frontend configuration + --rgw_arrow_flight start arrow flight frontend + --rgw_compression specify the rgw compression plugin + --seastore use seastore as crimson osd backend + -b, --bluestore use bluestore as the osd objectstore backend (default) + -K, --kstore use kstore as the osd objectstore backend + --cyanstore use cyanstore as the osd objectstore backend + --memstore use memstore as the osd objectstore backend + --cache <pool>: enable cache tiering on pool + --short: short object names only; necessary for ext4 dev + --nolockdep disable lockdep + --multimds <count> allow multimds with maximum active count + --without-dashboard: do not run using mgr dashboard + --bluestore-spdk: enable SPDK and with a comma-delimited list of PCI-IDs of NVME device (e.g, 0000:81:00.0) + --bluestore-pmem: enable PMEM and with path to a file mapped to PMEM + --msgr1: use msgr1 only + --msgr2: use msgr2 only + --msgr21: use msgr2 and msgr1 + --crimson: use crimson-osd instead of ceph-osd + --crimson-foreground: use crimson-osd, but run it in the foreground + --osd-args: specify any extra osd specific options + --bluestore-devs: comma-separated list of blockdevs to use for bluestore + --bluestore-zoned: blockdevs listed by --bluestore-devs are zoned devices (HM-SMR HDD or ZNS SSD) + --bluestore-io-uring: enable io_uring backend + --inc-osd: append some more osds into existing vcluster + --cephadm: enable cephadm orchestrator with ~/.ssh/id_rsa[.pub] + --no-parallel: dont start all OSDs in parallel + --no-restart: dont restart process when using ceph-run + --jaeger: use jaegertracing for tracing + --seastore-device-size: set total size of seastore + --seastore-devs: comma-separated list of blockdevs to use for seastore + --seastore-secondary-devs: comma-separated list of secondary blockdevs to use for seastore + --seastore-secondary-devs-type: device type of all secondary blockdevs. HDD, SSD(default), ZNS or RANDOM_BLOCK_SSD + --crimson-smp: number of cores to use for crimson +\n +EOF + +usage_exit() { + printf "$usage" + exit +} + +parse_block_devs() { + local opt_name=$1 + shift + local devs=$1 + shift + local dev + IFS=',' read -r -a block_devs <<< "$devs" + for dev in "${block_devs[@]}"; do + if [ ! -b $dev ] || [ ! -w $dev ]; then + echo "All $opt_name must refer to writable block devices" + exit 1 + fi + done +} + +parse_secondary_devs() { + local opt_name=$1 + shift + local devs=$1 + shift + local dev + IFS=',' read -r -a secondary_block_devs <<< "$devs" + for dev in "${secondary_block_devs[@]}"; do + if [ ! -b $dev ] || [ ! -w $dev ]; then + echo "All $opt_name must refer to writable block devices" + exit 1 + fi + done +} + +crimson_smp=1 +while [ $# -ge 1 ]; do +case $1 in + -d | --debug) + debug=1 + ;; + -t | --trace) + trace=1 + ;; + -s | --standby_mds) + standby=1 + ;; + -l | --localhost) + ip="127.0.0.1" + ;; + -i) + [ -z "$2" ] && usage_exit + ip="$2" + shift + ;; + -e) + ec=1 + ;; + --new | -n) + new=1 + ;; + --inc-osd) + new=0 + kill_all=0 + inc_osd_num=$2 + if [ "$inc_osd_num" == "" ]; then + inc_osd_num=1 + else + shift + fi + ;; + --short) + short=1 + ;; + --crimson) + crimson=1 + ceph_osd=crimson-osd + nodaemon=1 + msgr=2 + ;; + --crimson-foreground) + crimson=1 + ceph_osd=crimson-osd + nodaemon=0 + msgr=2 + ;; + --osd-args) + extra_osd_args="$2" + shift + ;; + --msgr1) + msgr="1" + ;; + --msgr2) + msgr="2" + ;; + --msgr21) + msgr="21" + ;; + --cephadm) + cephadm=1 + ;; + --no-parallel) + parallel=false + ;; + --no-restart) + restart=0 + ;; + --valgrind) + [ -z "$2" ] && usage_exit + valgrind=$2 + shift + ;; + --valgrind_args) + valgrind_args="$2" + shift + ;; + --valgrind_mds) + [ -z "$2" ] && usage_exit + valgrind_mds=$2 + shift + ;; + --valgrind_osd) + [ -z "$2" ] && usage_exit + valgrind_osd=$2 + shift + ;; + --valgrind_mon) + [ -z "$2" ] && usage_exit + valgrind_mon=$2 + shift + ;; + --valgrind_mgr) + [ -z "$2" ] && usage_exit + valgrind_mgr=$2 + shift + ;; + --valgrind_rgw) + [ -z "$2" ] && usage_exit + valgrind_rgw=$2 + shift + ;; + --nodaemon) + nodaemon=1 + ;; + --redirect-output) + redirect=1 + ;; + --smallmds) + smallmds=1 + ;; + --rgw_port) + CEPH_RGW_PORT=$2 + shift + ;; + --rgw_frontend) + rgw_frontend=$2 + shift + ;; + --rgw_arrow_flight) + rgw_flight_frontend="yes" + ;; + --rgw_compression) + rgw_compression=$2 + shift + ;; + --kstore_path) + kstore_path=$2 + shift + ;; + -m) + [ -z "$2" ] && usage_exit + MON_ADDR=$2 + shift + ;; + -x) + cephx=1 # this is on be default, flag exists for historical consistency + ;; + -X) + cephx=0 + ;; + + -g | --gssapi) + gssapi_authx=1 + ;; + -G) + gssapi_authx=0 + ;; + + -k) + if [ ! -r $conf_fn ]; then + echo "cannot use old configuration: $conf_fn not readable." >&2 + exit + fi + new=0 + ;; + --memstore) + objectstore="memstore" + ;; + --cyanstore) + objectstore="cyanstore" + ;; + --seastore) + objectstore="seastore" + ;; + -b | --bluestore) + objectstore="bluestore" + ;; + -K | --kstore) + objectstore="kstore" + ;; + --hitset) + hitset="$hitset $2 $3" + shift + shift + ;; + -o) + extra_conf+=$'\n'"$2" + shift + ;; + --cache) + if [ -z "$cache" ]; then + cache="$2" + else + cache="$cache $2" + fi + shift + ;; + --nolockdep) + lockdep=0 + ;; + --multimds) + CEPH_MAX_MDS="$2" + shift + ;; + --without-dashboard) + with_mgr_dashboard=false + ;; + --with-restful) + with_mgr_restful=true + ;; + --seastore-device-size) + seastore_size="$2" + shift + ;; + --seastore-devs) + parse_block_devs --seastore-devs "$2" + shift + ;; + --seastore-secondary-devs) + parse_secondary_devs --seastore-devs "$2" + shift + ;; + --seastore-secondary-devs-type) + secondary_block_devs_type="$2" + shift + ;; + --crimson-smp) + crimson_smp=$2 + shift + ;; + --bluestore-spdk) + [ -z "$2" ] && usage_exit + IFS=',' read -r -a bluestore_spdk_dev <<< "$2" + spdk_enabled=1 + shift + ;; + --bluestore-pmem) + [ -z "$2" ] && usage_exit + bluestore_pmem_file="$2" + pmem_enabled=1 + shift + ;; + --bluestore-devs) + parse_block_devs --bluestore-devs "$2" + shift + ;; + --bluestore-zoned) + zoned_enabled=1 + ;; + --bluestore-io-uring) + io_uring_enabled=1 + shift + ;; + --jaeger) + with_jaeger=1 + echo "with_jaeger $with_jaeger" + ;; + *) + usage_exit +esac +shift +done + +if [ $kill_all -eq 1 ]; then + $SUDO $INIT_CEPH stop +fi + +if [ "$new" -eq 0 ]; then + if [ -z "$CEPH_ASOK_DIR" ]; then + CEPH_ASOK_DIR=`dirname $($CEPH_BIN/ceph-conf -c $conf_fn --show-config-value admin_socket)` + fi + mkdir -p $CEPH_ASOK_DIR + MON=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC --lookup num_mon 2>/dev/null` && \ + CEPH_NUM_MON="$MON" + OSD=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC --lookup num_osd 2>/dev/null` && \ + CEPH_NUM_OSD="$OSD" + MDS=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC --lookup num_mds 2>/dev/null` && \ + CEPH_NUM_MDS="$MDS" + MGR=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC --lookup num_mgr 2>/dev/null` && \ + CEPH_NUM_MGR="$MGR" + RGW=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC --lookup num_rgw 2>/dev/null` && \ + CEPH_NUM_RGW="$RGW" + NFS=`$CEPH_BIN/ceph-conf -c $conf_fn --name $VSTART_SEC --lookup num_ganesha 2>/dev/null` && \ + GANESHA_DAEMON_NUM="$NFS" +else + # only delete if -n + if [ -e "$conf_fn" ]; then + asok_dir=`dirname $($CEPH_BIN/ceph-conf -c $conf_fn --show-config-value admin_socket)` + rm -- "$conf_fn" + if [ $asok_dir != /var/run/ceph ]; then + [ -d $asok_dir ] && rm -f $asok_dir/* && rmdir $asok_dir + fi + fi + if [ -z "$CEPH_ASOK_DIR" ]; then + CEPH_ASOK_DIR=`mktemp -u -d "${TMPDIR:-/tmp}/ceph-asok.XXXXXX"` + fi +fi + +ARGS="-c $conf_fn" + +run() { + type=$1 + shift + num=$1 + shift + eval "valg=\$valgrind_$type" + [ -z "$valg" ] && valg="$valgrind" + + if [ -n "$valg" ]; then + prunb valgrind --tool="$valg" $valgrind_args "$@" -f + sleep 1 + else + if [ "$nodaemon" -eq 0 ]; then + prun "$@" + else + if [ "$restart" -eq 0 ]; then + set -- '--no-restart' "$@" + fi + if [ "$redirect" -eq 0 ]; then + prunb ${CEPH_ROOT}/src/ceph-run "$@" -f + else + ( prunb ${CEPH_ROOT}/src/ceph-run "$@" -f ) >$CEPH_OUT_DIR/$type.$num.stdout 2>&1 + fi + fi + fi +} + +wconf() { + if [ "$new" -eq 1 -o "$overwrite_conf" -eq 1 ]; then + cat >> "$conf_fn" + fi +} + + +do_rgw_conf() { + + if [ $CEPH_NUM_RGW -eq 0 ]; then + return 0 + fi + + # setup each rgw on a sequential port, starting at $CEPH_RGW_PORT. + # individual rgw's ids will be their ports. + current_port=$CEPH_RGW_PORT + # allow only first rgw to start arrow_flight server/port + local flight_conf=$rgw_flight_frontend + for n in $(seq 1 $CEPH_NUM_RGW); do + wconf << EOF +[client.rgw.${current_port}] + rgw frontends = $rgw_frontend port=${current_port}${flight_conf:+,arrow_flight} + admin socket = ${CEPH_OUT_DIR}/radosgw.${current_port}.asok + debug rgw_flight = 20 +EOF + current_port=$((current_port + 1)) + unset flight_conf +done + +} + +format_conf() { + local opts=$1 + local indent=" " + local opt + local formatted + while read -r opt; do + if [ -z "$formatted" ]; then + formatted="${opt}" + else + formatted+=$'\n'${indent}${opt} + fi + done <<< "$opts" + echo "$formatted" +} + +prepare_conf() { + local DAEMONOPTS=" + log file = $CEPH_OUT_DIR/\$name.log + admin socket = $CEPH_ASOK_DIR/\$name.asok + chdir = \"\" + pid file = $CEPH_OUT_DIR/\$name.pid + heartbeat file = $CEPH_OUT_DIR/\$name.heartbeat +" + + local mgr_modules="iostat nfs" + if $with_mgr_dashboard; then + mgr_modules+=" dashboard" + fi + if $with_mgr_restful; then + mgr_modules+=" restful" + fi + + local msgr_conf='' + if [ $msgr -eq 21 ]; then + msgr_conf="ms bind msgr2 = true + ms bind msgr1 = true" + fi + if [ $msgr -eq 2 ]; then + msgr_conf="ms bind msgr2 = true + ms bind msgr1 = false" + fi + if [ $msgr -eq 1 ]; then + msgr_conf="ms bind msgr2 = false + ms bind msgr1 = true" + fi + + wconf <<EOF +; generated by vstart.sh on `date` +[$VSTART_SEC] + num mon = $CEPH_NUM_MON + num osd = $CEPH_NUM_OSD + num mds = $CEPH_NUM_MDS + num mgr = $CEPH_NUM_MGR + num rgw = $CEPH_NUM_RGW + num ganesha = $GANESHA_DAEMON_NUM + +[global] + fsid = $(uuidgen) + osd failsafe full ratio = .99 + mon osd full ratio = .99 + mon osd nearfull ratio = .99 + mon osd backfillfull ratio = .99 + mon_max_pg_per_osd = ${MON_MAX_PG_PER_OSD:-1000} + erasure code dir = $EC_PATH + plugin dir = $CEPH_LIB + run dir = $CEPH_OUT_DIR + crash dir = $CEPH_OUT_DIR + enable experimental unrecoverable data corrupting features = * + osd_crush_chooseleaf_type = 0 + debug asok assert abort = true + $(format_conf "${msgr_conf}") + $(format_conf "${extra_conf}") + $AUTOSCALER_OPTS +EOF + if [ "$with_jaeger" -eq 1 ] ; then + wconf <<EOF + jaeger_agent_port = 6831 +EOF + fi + if [ "$lockdep" -eq 1 ] ; then + wconf <<EOF + lockdep = true +EOF + fi + if [ "$cephx" -eq 1 ] ; then + wconf <<EOF + auth cluster required = cephx + auth service required = cephx + auth client required = cephx +EOF + elif [ "$gssapi_authx" -eq 1 ] ; then + wconf <<EOF + auth cluster required = gss + auth service required = gss + auth client required = gss + gss ktab client file = $CEPH_DEV_DIR/gss_\$name.keytab +EOF + else + wconf <<EOF + auth cluster required = none + auth service required = none + auth client required = none + ms mon client mode = crc +EOF + fi + if [ "$short" -eq 1 ]; then + COSDSHORT=" osd max object name len = 460 + osd max object namespace len = 64" + fi + if [ "$objectstore" == "bluestore" ]; then + if [ "$spdk_enabled" -eq 1 ] || [ "$pmem_enabled" -eq 1 ]; then + BLUESTORE_OPTS=" bluestore_block_db_path = \"\" + bluestore_block_db_size = 0 + bluestore_block_db_create = false + bluestore_block_wal_path = \"\" + bluestore_block_wal_size = 0 + bluestore_block_wal_create = false + bluestore_spdk_mem = 2048" + else + BLUESTORE_OPTS=" bluestore block db path = $CEPH_DEV_DIR/osd\$id/block.db.file + bluestore block db size = 1073741824 + bluestore block db create = true + bluestore block wal path = $CEPH_DEV_DIR/osd\$id/block.wal.file + bluestore block wal size = 1048576000 + bluestore block wal create = true" + fi + if [ "$zoned_enabled" -eq 1 ]; then + BLUESTORE_OPTS+=" + bluestore min alloc size = 65536 + bluestore prefer deferred size = 0 + bluestore prefer deferred size hdd = 0 + bluestore prefer deferred size ssd = 0 + bluestore allocator = zoned" + fi + if [ "$io_uring_enabled" -eq 1 ]; then + BLUESTORE_OPTS+=" + bdev ioring = true" + fi + fi + + if [ "$objectstore" == "seastore" ]; then + if [[ ${seastore_size+x} ]]; then + SEASTORE_OPTS=" + seastore device size = $seastore_size" + fi + fi + + wconf <<EOF +[client] +$CCLIENTDEBUG + keyring = $keyring_fn + log file = $CEPH_OUT_CLIENT_DIR/\$name.\$pid.log + admin socket = $CEPH_ASOK_DIR/\$name.\$pid.asok + + ; needed for s3tests + rgw crypt s3 kms backend = testing + rgw crypt s3 kms encryption keys = testkey-1=YmluCmJvb3N0CmJvb3N0LWJ1aWxkCmNlcGguY29uZgo= testkey-2=aWIKTWFrZWZpbGUKbWFuCm91dApzcmMKVGVzdGluZwo= + rgw crypt require ssl = false + ; uncomment the following to set LC days as the value in seconds; + ; needed for passing lc time based s3-tests (can be verbose) + ; rgw lc debug interval = 10 + $(format_conf "${extra_conf}") +EOF + do_rgw_conf + wconf << EOF +[mds] +$CMDSDEBUG +$DAEMONOPTS + mds data = $CEPH_DEV_DIR/mds.\$id + mds root ino uid = `id -u` + mds root ino gid = `id -g` + $(format_conf "${extra_conf}") +[mgr] + mgr disabled modules = rook + mgr data = $CEPH_DEV_DIR/mgr.\$id + mgr module path = $MGR_PYTHON_PATH + cephadm path = $CEPH_BIN/cephadm +$DAEMONOPTS + $(format_conf "${extra_conf}") +[osd] +$DAEMONOPTS + osd_check_max_object_name_len_on_startup = false + osd data = $CEPH_DEV_DIR/osd\$id + osd journal = $CEPH_DEV_DIR/osd\$id/journal + osd journal size = 100 + osd class tmp = out + osd class dir = $OBJCLASS_PATH + osd class load list = * + osd class default list = * + osd fast shutdown = false + + bluestore fsck on mount = true + bluestore block create = true +$BLUESTORE_OPTS + + ; kstore + kstore fsck on mount = true + osd objectstore = $objectstore +$SEASTORE_OPTS +$COSDSHORT + $(format_conf "${extra_conf}") +[mon] + mon_data_avail_crit = 1 + mgr initial modules = $mgr_modules +$DAEMONOPTS +$CMONDEBUG + $(format_conf "${extra_conf}") + mon cluster log file = $CEPH_OUT_DIR/cluster.mon.\$id.log + osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd + auth allow insecure global id reclaim = false +EOF + + if [ "$crimson" -eq 1 ]; then + wconf <<EOF + osd pool default crimson = true +EOF + fi +} + +write_logrotate_conf() { + out_dir=$(pwd)"/out/*.log" + + cat << EOF +$out_dir +{ + rotate 5 + size 1G + copytruncate + compress + notifempty + missingok + sharedscripts + postrotate + # NOTE: assuring that the absence of one of the following processes + # won't abort the logrotate command. + killall -u $USER -q -1 ceph-mon ceph-mgr ceph-mds ceph-osd ceph-fuse radosgw rbd-mirror || echo "" + endscript +} +EOF +} + +init_logrotate() { + logrotate_conf_path=$(pwd)"/logrotate.conf" + logrotate_state_path=$(pwd)"/logrotate.state" + + if ! test -a $logrotate_conf_path; then + if test -a $logrotate_state_path; then + rm -f $logrotate_state_path + fi + write_logrotate_conf > $logrotate_conf_path + fi +} + +start_mon() { + local MONS="" + local count=0 + for f in a b c d e f g h i j k l m n o p q r s t u v w x y z + do + [ $count -eq $CEPH_NUM_MON ] && break; + count=$(($count + 1)) + if [ -z "$MONS" ]; then + MONS="$f" + else + MONS="$MONS $f" + fi + done + + if [ "$new" -eq 1 ]; then + if [ `echo $IP | grep '^127\\.'` ]; then + echo + echo "NOTE: hostname resolves to loopback; remote hosts will not be able to" + echo " connect. either adjust /etc/hosts, or edit this script to use your" + echo " machine's real IP." + echo + fi + + prun $SUDO "$CEPH_BIN/ceph-authtool" --create-keyring --gen-key --name=mon. "$keyring_fn" --cap mon 'allow *' + prun $SUDO "$CEPH_BIN/ceph-authtool" --gen-key --name=client.admin \ + --cap mon 'allow *' \ + --cap osd 'allow *' \ + --cap mds 'allow *' \ + --cap mgr 'allow *' \ + "$keyring_fn" + + # build a fresh fs monmap, mon fs + local params=() + local count=0 + local mon_host="" + for f in $MONS + do + if [ $msgr -eq 1 ]; then + A="v1:$IP:$(($CEPH_PORT+$count+1))" + fi + if [ $msgr -eq 2 ]; then + A="v2:$IP:$(($CEPH_PORT+$count+1))" + fi + if [ $msgr -eq 21 ]; then + A="[v2:$IP:$(($CEPH_PORT+$count)),v1:$IP:$(($CEPH_PORT+$count+1))]" + fi + params+=("--addv" "$f" "$A") + mon_host="$mon_host $A" + wconf <<EOF +[mon.$f] + host = $HOSTNAME + mon data = $CEPH_DEV_DIR/mon.$f +EOF + count=$(($count + 2)) + done + wconf <<EOF +[global] + mon host = $mon_host +EOF + prun "$CEPH_BIN/monmaptool" --create --clobber "${params[@]}" --print "$monmap_fn" + + for f in $MONS + do + prun rm -rf -- "$CEPH_DEV_DIR/mon.$f" + prun mkdir -p "$CEPH_DEV_DIR/mon.$f" + prun "$CEPH_BIN/ceph-mon" --mkfs -c "$conf_fn" -i "$f" --monmap="$monmap_fn" --keyring="$keyring_fn" + done + + prun rm -- "$monmap_fn" + fi + + # start monitors + for f in $MONS + do + run 'mon' $f $CEPH_BIN/ceph-mon -i $f $ARGS $CMON_ARGS + done + + if [ "$crimson" -eq 1 ]; then + $CEPH_BIN/ceph osd set-allow-crimson --yes-i-really-mean-it + fi +} + +start_osd() { + if [ $inc_osd_num -gt 0 ]; then + old_maxosd=$($CEPH_BIN/ceph osd getmaxosd | sed -e 's/max_osd = //' -e 's/ in epoch.*//') + start=$old_maxosd + end=$(($start-1+$inc_osd_num)) + overwrite_conf=1 # fake wconf + else + start=0 + end=$(($CEPH_NUM_OSD-1)) + fi + local osds_wait + for osd in `seq $start $end` + do + local extra_seastar_args + if [ "$ceph_osd" == "crimson-osd" ]; then + bottom_cpu=$(( osd * crimson_smp )) + top_cpu=$(( bottom_cpu + crimson_smp - 1 )) + # set a single CPU nodes for each osd + extra_seastar_args="--cpuset $bottom_cpu-$top_cpu" + if [ "$debug" -ne 0 ]; then + extra_seastar_args+=" --debug" + fi + if [ "$trace" -ne 0 ]; then + extra_seastar_args+=" --trace" + fi + fi + if [ "$new" -eq 1 -o $inc_osd_num -gt 0 ]; then + wconf <<EOF +[osd.$osd] + host = $HOSTNAME +EOF + if [ "$spdk_enabled" -eq 1 ]; then + wconf <<EOF + bluestore_block_path = spdk:${bluestore_spdk_dev[$osd]} +EOF + elif [ "$pmem_enabled" -eq 1 ]; then + wconf <<EOF + bluestore_block_path = ${bluestore_pmem_file} +EOF + fi + rm -rf $CEPH_DEV_DIR/osd$osd || true + if command -v btrfs > /dev/null; then + for f in $CEPH_DEV_DIR/osd$osd/*; do btrfs sub delete $f &> /dev/null || true; done + fi + if [ -n "$kstore_path" ]; then + ln -s $kstore_path $CEPH_DEV_DIR/osd$osd + else + mkdir -p $CEPH_DEV_DIR/osd$osd + if [ -n "${block_devs[$osd]}" ]; then + dd if=/dev/zero of=${block_devs[$osd]} bs=1M count=1 + ln -s ${block_devs[$osd]} $CEPH_DEV_DIR/osd$osd/block + fi + if [ -n "${secondary_block_devs[$osd]}" ]; then + dd if=/dev/zero of=${secondary_block_devs[$osd]} bs=1M count=1 + mkdir -p $CEPH_DEV_DIR/osd$osd/block.${secondary_block_devs_type}.1 + ln -s ${secondary_block_devs[$osd]} $CEPH_DEV_DIR/osd$osd/block.${secondary_block_devs_type}.1/block + fi + fi + if [ "$objectstore" == "bluestore" ]; then + wconf <<EOF + bluestore fsck on mount = false +EOF + fi + + local uuid=`uuidgen` + echo "add osd$osd $uuid" + OSD_SECRET=$($CEPH_BIN/ceph-authtool --gen-print-key) + echo "{\"cephx_secret\": \"$OSD_SECRET\"}" > $CEPH_DEV_DIR/osd$osd/new.json + ceph_adm osd new $uuid -i $CEPH_DEV_DIR/osd$osd/new.json + rm $CEPH_DEV_DIR/osd$osd/new.json + prun $SUDO $CEPH_BIN/$ceph_osd $extra_osd_args -i $osd $ARGS --mkfs --key $OSD_SECRET --osd-uuid $uuid $extra_seastar_args \ + 2>&1 | tee $CEPH_OUT_DIR/osd-mkfs.$osd.log + + local key_fn=$CEPH_DEV_DIR/osd$osd/keyring + cat > $key_fn<<EOF +[osd.$osd] + key = $OSD_SECRET +EOF + fi + echo start osd.$osd + local osd_pid + echo 'osd' $osd $SUDO $CEPH_BIN/$ceph_osd \ + $extra_seastar_args $extra_osd_args \ + -i $osd $ARGS $COSD_ARGS + run 'osd' $osd $SUDO $CEPH_BIN/$ceph_osd \ + $extra_seastar_args $extra_osd_args \ + -i $osd $ARGS $COSD_ARGS & + osd_pid=$! + if $parallel; then + osds_wait=$osd_pid + else + wait $osd_pid + fi + done + if $parallel; then + for p in $osds_wait; do + wait $p + done + debug echo OSDs started + fi + if [ $inc_osd_num -gt 0 ]; then + # update num osd + new_maxosd=$($CEPH_BIN/ceph osd getmaxosd | sed -e 's/max_osd = //' -e 's/ in epoch.*//') + sed -i "s/num osd = .*/num osd = $new_maxosd/g" $conf_fn + fi +} + +create_mgr_restful_secret() { + while ! ceph_adm -h | grep -c -q ^restful ; do + debug echo 'waiting for mgr restful module to start' + sleep 1 + done + local secret_file + if ceph_adm restful create-self-signed-cert > /dev/null; then + secret_file=`mktemp` + ceph_adm restful create-key admin -o $secret_file + RESTFUL_SECRET=`cat $secret_file` + rm $secret_file + else + debug echo MGR Restful is not working, perhaps the package is not installed? + fi +} + +start_mgr() { + local mgr=0 + local ssl=${DASHBOARD_SSL:-1} + # avoid monitors on nearby ports (which test/*.sh use extensively) + MGR_PORT=$(($CEPH_PORT + 1000)) + PROMETHEUS_PORT=9283 + for name in x y z a b c d e f g h i j k l m n o p + do + [ $mgr -eq $CEPH_NUM_MGR ] && break + mgr=$(($mgr + 1)) + if [ "$new" -eq 1 ]; then + mkdir -p $CEPH_DEV_DIR/mgr.$name + key_fn=$CEPH_DEV_DIR/mgr.$name/keyring + $SUDO $CEPH_BIN/ceph-authtool --create-keyring --gen-key --name=mgr.$name $key_fn + ceph_adm -i $key_fn auth add mgr.$name mon 'allow profile mgr' mds 'allow *' osd 'allow *' + + wconf <<EOF +[mgr.$name] + host = $HOSTNAME +EOF + + if $with_mgr_dashboard ; then + local port_option="ssl_server_port" + local http_proto="https" + if [ "$ssl" == "0" ]; then + port_option="server_port" + http_proto="http" + ceph_adm config set mgr mgr/dashboard/ssl false --force + fi + ceph_adm config set mgr mgr/dashboard/$name/$port_option $MGR_PORT --force + if [ $mgr -eq 1 ]; then + DASH_URLS="$http_proto://$IP:$MGR_PORT" + else + DASH_URLS+=", $http_proto://$IP:$MGR_PORT" + fi + fi + MGR_PORT=$(($MGR_PORT + 1000)) + ceph_adm config set mgr mgr/prometheus/$name/server_port $PROMETHEUS_PORT --force + PROMETHEUS_PORT=$(($PROMETHEUS_PORT + 1000)) + + ceph_adm config set mgr mgr/restful/$name/server_port $MGR_PORT --force + if [ $mgr -eq 1 ]; then + RESTFUL_URLS="https://$IP:$MGR_PORT" + else + RESTFUL_URLS+=", https://$IP:$MGR_PORT" + fi + MGR_PORT=$(($MGR_PORT + 1000)) + fi + + debug echo "Starting mgr.${name}" + run 'mgr' $name $CEPH_BIN/ceph-mgr -i $name $ARGS + done + + while ! ceph_adm mgr stat | jq -e '.available'; do + debug echo 'waiting for mgr to become available' + sleep 1 + done + + if [ "$new" -eq 1 ]; then + # setting login credentials for dashboard + if $with_mgr_dashboard; then + while ! ceph_adm -h | grep -c -q ^dashboard ; do + debug echo 'waiting for mgr dashboard module to start' + sleep 1 + done + DASHBOARD_ADMIN_SECRET_FILE="${CEPH_CONF_PATH}/dashboard-admin-secret.txt" + printf 'admin' > "${DASHBOARD_ADMIN_SECRET_FILE}" + ceph_adm dashboard ac-user-create admin -i "${DASHBOARD_ADMIN_SECRET_FILE}" \ + administrator --force-password + if [ "$ssl" != "0" ]; then + if ! ceph_adm dashboard create-self-signed-cert; then + debug echo dashboard module not working correctly! + fi + fi + fi + if $with_mgr_restful; then + create_mgr_restful_secret + fi + fi + + if [ "$cephadm" -eq 1 ]; then + debug echo Enabling cephadm orchestrator + if [ "$new" -eq 1 ]; then + digest=$(curl -s \ + https://hub.docker.com/v2/repositories/ceph/daemon-base/tags/latest-master-devel \ + | jq -r '.images[0].digest') + ceph_adm config set global container_image "docker.io/ceph/daemon-base@$digest" + fi + ceph_adm config-key set mgr/cephadm/ssh_identity_key -i ~/.ssh/id_rsa + ceph_adm config-key set mgr/cephadm/ssh_identity_pub -i ~/.ssh/id_rsa.pub + ceph_adm mgr module enable cephadm + ceph_adm orch set backend cephadm + ceph_adm orch host add "$(hostname)" + ceph_adm orch apply crash '*' + ceph_adm config set mgr mgr/cephadm/allow_ptrace true + fi +} + +start_mds() { + local mds=0 + for name in a b c d e f g h i j k l m n o p + do + [ $mds -eq $CEPH_NUM_MDS ] && break + mds=$(($mds + 1)) + + if [ "$new" -eq 1 ]; then + prun mkdir -p "$CEPH_DEV_DIR/mds.$name" + key_fn=$CEPH_DEV_DIR/mds.$name/keyring + wconf <<EOF +[mds.$name] + host = $HOSTNAME +EOF + if [ "$standby" -eq 1 ]; then + mkdir -p $CEPH_DEV_DIR/mds.${name}s + wconf <<EOF + mds standby for rank = $mds +[mds.${name}s] + mds standby replay = true + mds standby for name = ${name} +EOF + fi + prun $SUDO "$CEPH_BIN/ceph-authtool" --create-keyring --gen-key --name="mds.$name" "$key_fn" + ceph_adm -i "$key_fn" auth add "mds.$name" mon 'allow profile mds' osd 'allow rw tag cephfs *=*' mds 'allow' mgr 'allow profile mds' + if [ "$standby" -eq 1 ]; then + prun $SUDO "$CEPH_BIN/ceph-authtool" --create-keyring --gen-key --name="mds.${name}s" \ + "$CEPH_DEV_DIR/mds.${name}s/keyring" + ceph_adm -i "$CEPH_DEV_DIR/mds.${name}s/keyring" auth add "mds.${name}s" \ + mon 'allow profile mds' osd 'allow *' mds 'allow' mgr 'allow profile mds' + fi + fi + + run 'mds' $name $CEPH_BIN/ceph-mds -i $name $ARGS $CMDS_ARGS + if [ "$standby" -eq 1 ]; then + run 'mds' $name $CEPH_BIN/ceph-mds -i ${name}s $ARGS $CMDS_ARGS + fi + + #valgrind --tool=massif $CEPH_BIN/ceph-mds $ARGS --mds_log_max_segments 2 --mds_thrash_fragments 0 --mds_thrash_exports 0 > m #--debug_ms 20 + #$CEPH_BIN/ceph-mds -d $ARGS --mds_thrash_fragments 0 --mds_thrash_exports 0 #--debug_ms 20 + #ceph_adm mds set max_mds 2 + done + + if [ $new -eq 1 ]; then + if [ "$CEPH_NUM_FS" -gt "0" ] ; then + sleep 5 # time for MDS to come up as standby to avoid health warnings on fs creation + if [ "$CEPH_NUM_FS" -gt "1" ] ; then + ceph_adm fs flag set enable_multiple true --yes-i-really-mean-it + fi + + # wait for volume module to load + while ! ceph_adm fs volume ls ; do sleep 1 ; done + local fs=0 + for name in a b c d e f g h i j k l m n o p + do + ceph_adm fs volume create ${name} + ceph_adm fs authorize ${name} "client.fs_${name}" / rwp >> "$keyring_fn" + fs=$(($fs + 1)) + [ $fs -eq $CEPH_NUM_FS ] && break + done + fi + fi + +} + +# Ganesha Daemons requires nfs-ganesha nfs-ganesha-ceph nfs-ganesha-rados-grace +# nfs-ganesha-rados-urls (version 3.3 and above) packages installed. On +# Fedora>=31 these packages can be installed directly with 'dnf'. For CentOS>=8 +# the packages are available at +# https://wiki.centos.org/SpecialInterestGroup/Storage +# Similarly for Ubuntu>=16.04 follow the instructions on +# https://launchpad.net/~nfs-ganesha + +start_ganesha() { + cluster_id="vstart" + GANESHA_PORT=$(($CEPH_PORT + 4000)) + local ganesha=0 + test_user="$cluster_id" + pool_name=".nfs" + namespace=$cluster_id + url="rados://$pool_name/$namespace/conf-nfs.$test_user" + + prun ceph_adm auth get-or-create client.$test_user \ + mon "allow r" \ + osd "allow rw pool=$pool_name namespace=$namespace, allow rw tag cephfs data=a" \ + mds "allow rw path=/" \ + >> "$keyring_fn" + + ceph_adm mgr module enable test_orchestrator + ceph_adm orch set backend test_orchestrator + ceph_adm test_orchestrator load_data -i $CEPH_ROOT/src/pybind/mgr/test_orchestrator/dummy_data.json + prun ceph_adm nfs cluster create $cluster_id + prun ceph_adm nfs export create cephfs --fsname "a" --cluster-id $cluster_id --pseudo-path "/cephfs" + + for name in a b c d e f g h i j k l m n o p + do + [ $ganesha -eq $GANESHA_DAEMON_NUM ] && break + + port=$(($GANESHA_PORT + ganesha)) + ganesha=$(($ganesha + 1)) + ganesha_dir="$CEPH_DEV_DIR/ganesha.$name" + prun rm -rf $ganesha_dir + prun mkdir -p $ganesha_dir + + echo "NFS_CORE_PARAM { + Enable_NLM = false; + Enable_RQUOTA = false; + Protocols = 4; + NFS_Port = $port; + } + + MDCACHE { + Dir_Chunk = 0; + } + + NFSv4 { + RecoveryBackend = rados_cluster; + Minor_Versions = 1, 2; + } + + RADOS_KV { + pool = '$pool_name'; + namespace = $namespace; + UserId = $test_user; + nodeid = $name; + } + + RADOS_URLS { + Userid = $test_user; + watch_url = '$url'; + } + + %url $url" > "$ganesha_dir/ganesha-$name.conf" + wconf <<EOF +[ganesha.$name] + host = $HOSTNAME + ip = $IP + port = $port + ganesha data = $ganesha_dir + pid file = $CEPH_OUT_DIR/ganesha-$name.pid +EOF + + prun env CEPH_CONF="${conf_fn}" ganesha-rados-grace --userid $test_user -p $pool_name -n $namespace add $name + prun env CEPH_CONF="${conf_fn}" ganesha-rados-grace --userid $test_user -p $pool_name -n $namespace + + prun env CEPH_CONF="${conf_fn}" ganesha.nfsd -L "$CEPH_OUT_DIR/ganesha-$name.log" -f "$ganesha_dir/ganesha-$name.conf" -p "$CEPH_OUT_DIR/ganesha-$name.pid" -N NIV_DEBUG + + # Wait few seconds for grace period to be removed + sleep 2 + + prun env CEPH_CONF="${conf_fn}" ganesha-rados-grace --userid $test_user -p $pool_name -n $namespace + + echo "$test_user ganesha daemon $name started on port: $port" + done +} + +if [ "$debug" -eq 0 ]; then + CMONDEBUG=' + debug mon = 10 + debug ms = 1' + CCLIENTDEBUG='' + CMDSDEBUG='' +else + debug echo "** going verbose **" + CMONDEBUG=' + debug osd = 20 + debug mon = 20 + debug osd = 20 + debug paxos = 20 + debug auth = 20 + debug mgrc = 20 + debug ms = 1' + CCLIENTDEBUG=' + debug client = 20' + CMDSDEBUG=' + debug mds = 20' +fi + +# Crimson doesn't support PG merge/split yet. +if [ "$ceph_osd" == "crimson-osd" ]; then + AUTOSCALER_OPTS=' + osd_pool_default_pg_autoscale_mode = off' +fi + +if [ -n "$MON_ADDR" ]; then + CMON_ARGS=" -m "$MON_ADDR + COSD_ARGS=" -m "$MON_ADDR + CMDS_ARGS=" -m "$MON_ADDR +fi + +if [ -z "$CEPH_PORT" ]; then + while [ true ] + do + CEPH_PORT="$(echo $(( RANDOM % 1000 + 40000 )))" + ss -a -n | egrep "\<LISTEN\>.+:${CEPH_PORT}\s+" 1>/dev/null 2>&1 || break + done +fi + +[ -z "$INIT_CEPH" ] && INIT_CEPH=$CEPH_BIN/init-ceph + +# sudo if btrfs +[ -d $CEPH_DEV_DIR/osd0/. ] && [ -e $CEPH_DEV_DIR/sudo ] && SUDO="sudo" + +if [ $inc_osd_num -eq 0 ]; then + prun $SUDO rm -f core* +fi + +[ -d $CEPH_ASOK_DIR ] || mkdir -p $CEPH_ASOK_DIR +[ -d $CEPH_OUT_DIR ] || mkdir -p $CEPH_OUT_DIR +[ -d $CEPH_DEV_DIR ] || mkdir -p $CEPH_DEV_DIR +[ -d $CEPH_OUT_CLIENT_DIR ] || mkdir -p $CEPH_OUT_CLIENT_DIR +if [ $inc_osd_num -eq 0 ]; then + $SUDO find "$CEPH_OUT_DIR" -type f -delete +fi +[ -d gmon ] && $SUDO rm -rf gmon/* + +[ "$cephx" -eq 1 ] && [ "$new" -eq 1 ] && [ -e $keyring_fn ] && rm $keyring_fn + + +# figure machine's ip +HOSTNAME=`hostname -s` +if [ -n "$ip" ]; then + IP="$ip" +else + echo hostname $HOSTNAME + if [ -x "$(which ip 2>/dev/null)" ]; then + IP_CMD="ip addr" + else + IP_CMD="ifconfig" + fi + # filter out IPv4 and localhost addresses + IP="$($IP_CMD | sed -En 's/127.0.0.1//;s/.*inet (addr:)?(([0-9]*\.){3}[0-9]*).*/\2/p' | head -n1)" + # if nothing left, try using localhost address, it might work + if [ -z "$IP" ]; then IP="127.0.0.1"; fi +fi +echo "ip $IP" +echo "port $CEPH_PORT" + + +[ -z $CEPH_ADM ] && CEPH_ADM=$CEPH_BIN/ceph + +ceph_adm() { + if [ "$cephx" -eq 1 ]; then + prun $SUDO "$CEPH_ADM" -c "$conf_fn" -k "$keyring_fn" "$@" + else + prun $SUDO "$CEPH_ADM" -c "$conf_fn" "$@" + fi +} + +if [ $inc_osd_num -gt 0 ]; then + start_osd + exit +fi + +if [ "$new" -eq 1 ]; then + prepare_conf +fi + +if [ $CEPH_NUM_MON -gt 0 ]; then + start_mon + + debug echo Populating config ... + cat <<EOF | $CEPH_BIN/ceph -c $conf_fn config assimilate-conf -i - +[global] +osd_pool_default_size = $OSD_POOL_DEFAULT_SIZE +osd_pool_default_min_size = 1 + +[mon] +mon_osd_reporter_subtree_level = osd +mon_data_avail_warn = 2 +mon_data_avail_crit = 1 +mon_allow_pool_delete = true +mon_allow_pool_size_one = true + +[osd] +osd_scrub_load_threshold = 2000 +osd_debug_op_order = true +osd_debug_misdirected_ops = true +osd_copyfrom_max_chunk = 524288 + +[mds] +mds_debug_frag = true +mds_debug_auth_pins = true +mds_debug_subtrees = true + +[mgr] +mgr/telemetry/nag = false +mgr/telemetry/enable = false + +EOF + + if [ "$debug" -ne 0 ]; then + debug echo Setting debug configs ... + cat <<EOF | $CEPH_BIN/ceph -c $conf_fn config assimilate-conf -i - +[mgr] +debug_ms = 1 +debug_mgr = 20 +debug_monc = 20 +debug_mon = 20 + +[osd] +debug_ms = 1 +debug_osd = 25 +debug_objecter = 20 +debug_monc = 20 +debug_mgrc = 20 +debug_journal = 20 +debug_bluestore = 20 +debug_bluefs = 20 +debug_rocksdb = 20 +debug_bdev = 20 +debug_reserver = 10 +debug_objclass = 20 + +[mds] +debug_ms = 1 +debug_mds = 20 +debug_monc = 20 +debug_mgrc = 20 +mds_debug_scatterstat = true +mds_verify_scatter = true +EOF + fi + if [ "$cephadm" -gt 0 ]; then + debug echo Setting mon public_network ... + public_network=$(ip route list | grep -w "$IP" | awk '{print $1}') + ceph_adm config set mon public_network $public_network + fi +fi + +if [ "$ceph_osd" == "crimson-osd" ]; then + $CEPH_BIN/ceph -c $conf_fn config set osd crimson_seastar_smp $crimson_smp +fi + +if [ $CEPH_NUM_MGR -gt 0 ]; then + start_mgr +fi + +# osd +if [ $CEPH_NUM_OSD -gt 0 ]; then + start_osd +fi + +# mds +if [ "$smallmds" -eq 1 ]; then + wconf <<EOF +[mds] + mds log max segments = 2 + # Default 'mds cache memory limit' is 1GiB, and here we set it to 100MiB. + mds cache memory limit = 100M +EOF +fi + +if [ $CEPH_NUM_MDS -gt 0 ]; then + start_mds + # key with access to all FS + ceph_adm fs authorize \* "client.fs" / rwp >> "$keyring_fn" +fi + +# Don't set max_mds until all the daemons are started, otherwise +# the intended standbys might end up in active roles. +if [ "$CEPH_MAX_MDS" -gt 1 ]; then + sleep 5 # wait for daemons to make it into FSMap before increasing max_mds +fi +fs=0 +for name in a b c d e f g h i j k l m n o p +do + [ $fs -eq $CEPH_NUM_FS ] && break + fs=$(($fs + 1)) + if [ "$CEPH_MAX_MDS" -gt 1 ]; then + ceph_adm fs set "${name}" max_mds "$CEPH_MAX_MDS" + fi +done + +# mgr + +if [ "$ec" -eq 1 ]; then + ceph_adm <<EOF +osd erasure-code-profile set ec-profile m=2 k=2 +osd pool create ec erasure ec-profile +EOF +fi + +do_cache() { + while [ -n "$*" ]; do + p="$1" + shift + debug echo "creating cache for pool $p ..." + ceph_adm <<EOF +osd pool create ${p}-cache +osd tier add $p ${p}-cache +osd tier cache-mode ${p}-cache writeback +osd tier set-overlay $p ${p}-cache +EOF + done +} +do_cache $cache + +do_hitsets() { + while [ -n "$*" ]; do + pool="$1" + type="$2" + shift + shift + debug echo "setting hit_set on pool $pool type $type ..." + ceph_adm <<EOF +osd pool set $pool hit_set_type $type +osd pool set $pool hit_set_count 8 +osd pool set $pool hit_set_period 30 +EOF + done +} +do_hitsets $hitset + +do_rgw_create_bucket() +{ + # Create RGW Bucket + local rgw_python_file='rgw-create-bucket.py' + echo "import boto +import boto.s3.connection + +conn = boto.connect_s3( + aws_access_key_id = '$s3_akey', + aws_secret_access_key = '$s3_skey', + host = '$HOSTNAME', + port = 80, + is_secure=False, + calling_format = boto.s3.connection.OrdinaryCallingFormat(), + ) + +bucket = conn.create_bucket('nfs-bucket') +print('created new bucket')" > "$CEPH_OUT_DIR/$rgw_python_file" + prun python $CEPH_OUT_DIR/$rgw_python_file +} + +do_rgw_create_users() +{ + # Create S3 user + s3_akey='0555b35654ad1656d804' + s3_skey='h7GhxuBLTrlhVUyxSPUKUV8r/2EI4ngqJxD7iBdBYLhwluN30JaT3Q==' + debug echo "setting up user testid" + $CEPH_BIN/radosgw-admin user create --uid testid --access-key $s3_akey --secret $s3_skey --display-name 'M. Tester' --email tester@ceph.com -c $conf_fn > /dev/null + + # Create S3-test users + # See: https://github.com/ceph/s3-tests + debug echo "setting up s3-test users" + $CEPH_BIN/radosgw-admin user create \ + --uid 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef \ + --access-key ABCDEFGHIJKLMNOPQRST \ + --secret abcdefghijklmnopqrstuvwxyzabcdefghijklmn \ + --display-name youruseridhere \ + --email s3@example.com --caps="user-policy=*" -c $conf_fn > /dev/null + $CEPH_BIN/radosgw-admin user create \ + --uid 56789abcdef0123456789abcdef0123456789abcdef0123456789abcdef01234 \ + --access-key NOPQRSTUVWXYZABCDEFG \ + --secret nopqrstuvwxyzabcdefghijklmnabcdefghijklm \ + --display-name john.doe \ + --email john.doe@example.com -c $conf_fn > /dev/null + $CEPH_BIN/radosgw-admin user create \ + --tenant testx \ + --uid 9876543210abcdef0123456789abcdef0123456789abcdef0123456789abcdef \ + --access-key HIJKLMNOPQRSTUVWXYZA \ + --secret opqrstuvwxyzabcdefghijklmnopqrstuvwxyzab \ + --display-name tenanteduser \ + --email tenanteduser@example.com -c $conf_fn > /dev/null + + # Create Swift user + debug echo "setting up user tester" + $CEPH_BIN/radosgw-admin user create -c $conf_fn --subuser=test:tester --display-name=Tester-Subuser --key-type=swift --secret=testing --access=full > /dev/null + + echo "" + echo "S3 User Info:" + echo " access key: $s3_akey" + echo " secret key: $s3_skey" + echo "" + echo "Swift User Info:" + echo " account : test" + echo " user : tester" + echo " password : testing" + echo "" +} + +do_rgw() +{ + if [ "$new" -eq 1 ]; then + do_rgw_create_users + if [ -n "$rgw_compression" ]; then + debug echo "setting compression type=$rgw_compression" + $CEPH_BIN/radosgw-admin zone placement modify -c $conf_fn --rgw-zone=default --placement-id=default-placement --compression=$rgw_compression > /dev/null + fi + fi + + if [ -n "$rgw_flight_frontend" ] ;then + debug echo "starting arrow_flight frontend on first rgw" + fi + + # Start server + if [ "$cephadm" -gt 0 ]; then + ceph_adm orch apply rgw rgwTest + return + fi + + RGWDEBUG="" + if [ "$debug" -ne 0 ]; then + RGWDEBUG="--debug-rgw=20 --debug-ms=1" + fi + + local CEPH_RGW_PORT_NUM="${CEPH_RGW_PORT}" + local CEPH_RGW_HTTPS="${CEPH_RGW_PORT: -1}" + if [[ "${CEPH_RGW_HTTPS}" = "s" ]]; then + CEPH_RGW_PORT_NUM="${CEPH_RGW_PORT::-1}" + else + CEPH_RGW_HTTPS="" + fi + RGWSUDO= + [ $CEPH_RGW_PORT_NUM -lt 1024 ] && RGWSUDO=sudo + + current_port=$CEPH_RGW_PORT + # allow only first rgw to start arrow_flight server/port + local flight_conf=$rgw_flight_frontend + for n in $(seq 1 $CEPH_NUM_RGW); do + rgw_name="client.rgw.${current_port}" + + ceph_adm auth get-or-create $rgw_name \ + mon 'allow rw' \ + osd 'allow rwx' \ + mgr 'allow rw' \ + >> "$keyring_fn" + + debug echo start rgw on http${CEPH_RGW_HTTPS}://localhost:${current_port} + run 'rgw' $current_port $RGWSUDO $CEPH_BIN/radosgw -c $conf_fn \ + --log-file=${CEPH_OUT_DIR}/radosgw.${current_port}.log \ + --admin-socket=${CEPH_OUT_DIR}/radosgw.${current_port}.asok \ + --pid-file=${CEPH_OUT_DIR}/radosgw.${current_port}.pid \ + --rgw_luarocks_location=${CEPH_OUT_DIR}/luarocks \ + ${RGWDEBUG} \ + -n ${rgw_name} \ + "--rgw_frontends=${rgw_frontend} port=${current_port}${CEPH_RGW_HTTPS}${flight_conf:+,arrow_flight}" + + i=$(($i + 1)) + [ $i -eq $CEPH_NUM_RGW ] && break + + current_port=$((current_port+1)) + unset flight_conf + done +} +if [ "$CEPH_NUM_RGW" -gt 0 ]; then + do_rgw +fi + +# Ganesha Daemons +if [ $GANESHA_DAEMON_NUM -gt 0 ]; then + pseudo_path="/cephfs" + if [ "$cephadm" -gt 0 ]; then + cluster_id="vstart" + port="2049" + prun ceph_adm nfs cluster create $cluster_id + if [ $CEPH_NUM_MDS -gt 0 ]; then + prun ceph_adm nfs export create cephfs --fsname "a" --cluster-id $cluster_id --pseudo-path $pseudo_path + echo "Mount using: mount -t nfs -o port=$port $IP:$pseudo_path mountpoint" + fi + if [ "$CEPH_NUM_RGW" -gt 0 ]; then + pseudo_path="/rgw" + do_rgw_create_bucket + prun ceph_adm nfs export create rgw --cluster-id $cluster_id --pseudo-path $pseudo_path --bucket "nfs-bucket" + echo "Mount using: mount -t nfs -o port=$port $IP:$pseudo_path mountpoint" + fi + else + start_ganesha + echo "Mount using: mount -t nfs -o port=<ganesha-port-num> $IP:$pseudo_path mountpoint" + fi +fi + +docker_service(){ + local service='' + #prefer podman + if command -v podman > /dev/null; then + service="podman" + elif pgrep -f docker > /dev/null; then + service="docker" + fi + if [ -n "$service" ]; then + echo "using $service for deploying jaeger..." + #check for exited container, remove them and restart container + if [ "$($service ps -aq -f status=exited -f name=jaeger)" ]; then + $service rm jaeger + fi + if [ ! "$(podman ps -aq -f name=jaeger)" ]; then + $service "$@" + fi + else + echo "cannot find docker or podman, please restart service and rerun." + fi +} + +echo "" +if [ $with_jaeger -eq 1 ]; then + debug echo "Enabling jaegertracing..." + docker_service run -d --name jaeger \ + -p 5775:5775/udp \ + -p 6831:6831/udp \ + -p 6832:6832/udp \ + -p 5778:5778 \ + -p 16686:16686 \ + -p 14268:14268 \ + -p 14250:14250 \ + quay.io/jaegertracing/all-in-one +fi + +debug echo "vstart cluster complete. Use stop.sh to stop. See out/* (e.g. 'tail -f out/????') for debug output." + +echo "" +if [ "$new" -eq 1 ]; then + if $with_mgr_dashboard; then + cat <<EOF +dashboard urls: $DASH_URLS + w/ user/pass: admin / admin +EOF + fi + if $with_mgr_restful; then + cat <<EOF +restful urls: $RESTFUL_URLS + w/ user/pass: admin / $RESTFUL_SECRET +EOF + fi +fi + +echo "" +# add header to the environment file +{ + echo "#" + echo "# source this file into your shell to set up the environment." + echo "# For example:" + echo "# $ . $CEPH_DIR/vstart_environment.sh" + echo "#" +} > $CEPH_DIR/vstart_environment.sh +{ + echo "export PYTHONPATH=$PYBIND:$CYTHON_PYTHONPATH:$CEPH_PYTHON_COMMON\$PYTHONPATH" + echo "export LD_LIBRARY_PATH=$CEPH_LIB:\$LD_LIBRARY_PATH" + echo "export PATH=$CEPH_DIR/bin:\$PATH" + + if [ "$CEPH_DIR" != "$PWD" ]; then + echo "export CEPH_CONF=$conf_fn" + echo "export CEPH_KEYRING=$keyring_fn" + fi + + if [ -n "$CEPHFS_SHELL" ]; then + echo "alias cephfs-shell=$CEPHFS_SHELL" + fi +} | tee -a $CEPH_DIR/vstart_environment.sh + +echo "CEPH_DEV=1" + +# always keep this section at the very bottom of this file +STRAY_CONF_PATH="/etc/ceph/ceph.conf" +if [ -f "$STRAY_CONF_PATH" -a -n "$conf_fn" -a ! "$conf_fn" -ef "$STRAY_CONF_PATH" ]; then + echo "" + echo "" + echo "WARNING:" + echo " Please remove stray $STRAY_CONF_PATH if not needed." + echo " Your conf files $conf_fn and $STRAY_CONF_PATH may not be in sync" + echo " and may lead to undesired results." + echo "" + echo "NOTE:" + echo " Remember to restart cluster after removing $STRAY_CONF_PATH" +fi + +init_logrotate |