diff options
Diffstat (limited to 'qa/workunits/cephadm')
| -rwxr-xr-x | qa/workunits/cephadm/create_iscsi_disks.sh | 36 | ||||
| -rwxr-xr-x | qa/workunits/cephadm/test_adoption.sh | 60 | ||||
| -rwxr-xr-x | qa/workunits/cephadm/test_cephadm.sh | 474 | ||||
| -rwxr-xr-x | qa/workunits/cephadm/test_dashboard_e2e.sh | 107 | ||||
| -rwxr-xr-x | qa/workunits/cephadm/test_iscsi_etc_hosts.sh | 21 | ||||
| -rwxr-xr-x | qa/workunits/cephadm/test_iscsi_pids_limit.sh | 29 | ||||
| -rwxr-xr-x | qa/workunits/cephadm/test_repos.sh | 45 |
7 files changed, 772 insertions, 0 deletions
diff --git a/qa/workunits/cephadm/create_iscsi_disks.sh b/qa/workunits/cephadm/create_iscsi_disks.sh new file mode 100755 index 000000000..45319e3a1 --- /dev/null +++ b/qa/workunits/cephadm/create_iscsi_disks.sh @@ -0,0 +1,36 @@ +#!/bin/bash -ex +# Create some file-backed iSCSI targets and attach them locally. + +# Exit if it's not CentOS +if ! grep -q rhel /etc/*-release; then + echo "The script only supports CentOS." + exit 1 +fi + +[ -z "$SUDO" ] && SUDO=sudo + +# 15 GB +DISK_FILE_SIZE="16106127360" + +$SUDO yum install -y targetcli iscsi-initiator-utils + +TARGET_NAME="iqn.2003-01.org.linux-iscsi.$(hostname).x8664:sn.foobar" +$SUDO targetcli /iscsi create ${TARGET_NAME} +$SUDO targetcli /iscsi/${TARGET_NAME}/tpg1/portals delete 0.0.0.0 3260 +$SUDO targetcli /iscsi/${TARGET_NAME}/tpg1/portals create 127.0.0.1 3260 +$SUDO targetcli /iscsi/${TARGET_NAME}/tpg1 set attribute generate_node_acls=1 +$SUDO targetcli /iscsi/${TARGET_NAME}/tpg1 set attribute demo_mode_write_protect=0 + +for i in $(seq 3); do + # Create truncated files, and add them as luns + DISK_FILE="/tmp/disk${i}" + $SUDO truncate --size ${DISK_FILE_SIZE} ${DISK_FILE} + + $SUDO targetcli /backstores/fileio create "lun${i}" ${DISK_FILE} + # Workaround for https://tracker.ceph.com/issues/47758 + $SUDO targetcli "/backstores/fileio/lun${i}" set attribute optimal_sectors=0 + $SUDO targetcli /iscsi/${TARGET_NAME}/tpg1/luns create "/backstores/fileio/lun${i}" +done + +$SUDO iscsiadm -m discovery -t sendtargets -p 127.0.0.1 +$SUDO iscsiadm -m node -p 127.0.0.1 -T ${TARGET_NAME} -l diff --git a/qa/workunits/cephadm/test_adoption.sh b/qa/workunits/cephadm/test_adoption.sh new file mode 100755 index 000000000..68580eb62 --- /dev/null +++ b/qa/workunits/cephadm/test_adoption.sh @@ -0,0 +1,60 @@ +#!/bin/bash -ex + +SCRIPT_NAME=$(basename ${BASH_SOURCE[0]}) +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +CEPHADM_SRC_DIR=${SCRIPT_DIR}/../../../src/cephadm +CORPUS_COMMIT=9cd9ad020d93b0b420924fec55da307aff8bd422 + +[ -z "$SUDO" ] && SUDO=sudo + +[ -d "$TMPDIR" ] || TMPDIR=$(mktemp -d tmp.$SCRIPT_NAME.XXXXXX) +trap "$SUDO rm -rf $TMPDIR" EXIT + +if [ -z "$CEPHADM" ]; then + CEPHADM=`mktemp -p $TMPDIR tmp.cephadm.XXXXXX` + ${CEPHADM_SRC_DIR}/build.sh "$CEPHADM" +fi + +# at this point, we need $CEPHADM set +if ! [ -x "$CEPHADM" ]; then + echo "cephadm not found. Please set \$CEPHADM" + exit 1 +fi + +# combine into a single var +CEPHADM_BIN="$CEPHADM" +CEPHADM="$SUDO $CEPHADM_BIN" + +## adopt +CORPUS_GIT_SUBMOD="cephadm-adoption-corpus" +GIT_CLONE_DIR=${TMPDIR}/${CORPUS_GIT_SUBMOD} +git clone https://github.com/ceph/$CORPUS_GIT_SUBMOD $GIT_CLONE_DIR + +git -C $GIT_CLONE_DIR checkout $CORPUS_COMMIT +CORPUS_DIR=${GIT_CLONE_DIR}/archive + +for subdir in `ls ${CORPUS_DIR}`; do + for tarfile in `ls ${CORPUS_DIR}/${subdir} | grep .tgz`; do + tarball=${CORPUS_DIR}/${subdir}/${tarfile} + FSID_LEGACY=`echo "$tarfile" | cut -c 1-36` + TMP_TAR_DIR=`mktemp -d -p $TMPDIR` + $SUDO tar xzvf $tarball -C $TMP_TAR_DIR + NAMES=$($CEPHADM ls --legacy-dir $TMP_TAR_DIR | jq -r '.[].name') + for name in $NAMES; do + $CEPHADM adopt \ + --style legacy \ + --legacy-dir $TMP_TAR_DIR \ + --name $name + # validate after adopt + out=$($CEPHADM ls | jq '.[]' \ + | jq 'select(.name == "'$name'")') + echo $out | jq -r '.style' | grep 'cephadm' + echo $out | jq -r '.fsid' | grep $FSID_LEGACY + done + # clean-up before next iter + $CEPHADM rm-cluster --fsid $FSID_LEGACY --force + $SUDO rm -rf $TMP_TAR_DIR + done +done + +echo "OK" diff --git a/qa/workunits/cephadm/test_cephadm.sh b/qa/workunits/cephadm/test_cephadm.sh new file mode 100755 index 000000000..7d06a3326 --- /dev/null +++ b/qa/workunits/cephadm/test_cephadm.sh @@ -0,0 +1,474 @@ +#!/bin/bash -ex + +SCRIPT_NAME=$(basename ${BASH_SOURCE[0]}) +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" + +# cleanup during exit +[ -z "$CLEANUP" ] && CLEANUP=true + +FSID='00000000-0000-0000-0000-0000deadbeef' + +# images that are used +IMAGE_MAIN=${IMAGE_MAIN:-'quay.ceph.io/ceph-ci/ceph:main'} +IMAGE_PACIFIC=${IMAGE_PACIFIC:-'quay.ceph.io/ceph-ci/ceph:pacific'} +#IMAGE_OCTOPUS=${IMAGE_OCTOPUS:-'quay.ceph.io/ceph-ci/ceph:octopus'} +IMAGE_DEFAULT=${IMAGE_MAIN} + +OSD_IMAGE_NAME="${SCRIPT_NAME%.*}_osd.img" +OSD_IMAGE_SIZE='6G' +OSD_TO_CREATE=2 +OSD_VG_NAME=${SCRIPT_NAME%.*} +OSD_LV_NAME=${SCRIPT_NAME%.*} + +# TMPDIR for test data +[ -d "$TMPDIR" ] || TMPDIR=$(mktemp -d tmp.$SCRIPT_NAME.XXXXXX) +[ -d "$TMPDIR_TEST_MULTIPLE_MOUNTS" ] || TMPDIR_TEST_MULTIPLE_MOUNTS=$(mktemp -d tmp.$SCRIPT_NAME.XXXXXX) + +CEPHADM_SRC_DIR=${SCRIPT_DIR}/../../../src/cephadm +CEPHADM_SAMPLES_DIR=${CEPHADM_SRC_DIR}/samples + +[ -z "$SUDO" ] && SUDO=sudo + +# If cephadm is already installed on the system, use that one, avoid building +# # one if we can. +if [ -z "$CEPHADM" ] && command -v cephadm >/dev/null ; then + CEPHADM="$(command -v cephadm)" +fi + +if [ -z "$CEPHADM" ]; then + CEPHADM=`mktemp -p $TMPDIR tmp.cephadm.XXXXXX` + ${CEPHADM_SRC_DIR}/build.sh "$CEPHADM" + NO_BUILD_INFO=1 +fi + +# at this point, we need $CEPHADM set +if ! [ -x "$CEPHADM" ]; then + echo "cephadm not found. Please set \$CEPHADM" + exit 1 +fi + +# add image to args +CEPHADM_ARGS="$CEPHADM_ARGS --image $IMAGE_DEFAULT" + +# combine into a single var +CEPHADM_BIN="$CEPHADM" +CEPHADM="$SUDO $CEPHADM_BIN $CEPHADM_ARGS" + +# clean up previous run(s)? +$CEPHADM rm-cluster --fsid $FSID --force +$SUDO vgchange -an $OSD_VG_NAME || true +loopdev=$($SUDO losetup -a | grep $(basename $OSD_IMAGE_NAME) | awk -F : '{print $1}') +if ! [ "$loopdev" = "" ]; then + $SUDO losetup -d $loopdev +fi + +function cleanup() +{ + if [ $CLEANUP = false ]; then + # preserve the TMPDIR state + echo "========================" + echo "!!! CLEANUP=$CLEANUP !!!" + echo + echo "TMPDIR=$TMPDIR" + echo "========================" + return + fi + + dump_all_logs $FSID + rm -rf $TMPDIR +} +trap cleanup EXIT + +function expect_false() +{ + set -x + if eval "$@"; then return 1; else return 0; fi +} + +# expect_return_code $expected_code $command ... +function expect_return_code() +{ + set -x + local expected_code="$1" + shift + local command="$@" + + set +e + eval "$command" + local return_code="$?" + set -e + + if [ ! "$return_code" -eq "$expected_code" ]; then return 1; else return 0; fi +} + +function is_available() +{ + local name="$1" + local condition="$2" + local tries="$3" + + local num=0 + while ! eval "$condition"; do + num=$(($num + 1)) + if [ "$num" -ge $tries ]; then + echo "$name is not available" + false + fi + sleep 5 + done + + echo "$name is available" + true +} + +function dump_log() +{ + local fsid="$1" + local name="$2" + local num_lines="$3" + + if [ -z $num_lines ]; then + num_lines=100 + fi + + echo '-------------------------' + echo 'dump daemon log:' $name + echo '-------------------------' + + $CEPHADM logs --fsid $fsid --name $name -- --no-pager -n $num_lines +} + +function dump_all_logs() +{ + local fsid="$1" + local names=$($CEPHADM ls | jq -r '.[] | select(.fsid == "'$fsid'").name') + + echo 'dumping logs for daemons: ' $names + for name in $names; do + dump_log $fsid $name + done +} + +function nfs_stop() +{ + # stop the running nfs server + local units="nfs-server nfs-kernel-server" + for unit in $units; do + if systemctl --no-pager status $unit > /dev/null; then + $SUDO systemctl stop $unit + fi + done + + # ensure the NFS port is no longer in use + expect_false "$SUDO ss -tlnp '( sport = :nfs )' | grep LISTEN" +} + +## prepare + check host +$SUDO $CEPHADM check-host + +## run a gather-facts (output to stdout) +$SUDO $CEPHADM gather-facts + +## NOTE: cephadm version is, as of around May 2023, no longer basing the +## output for `cephadm version` on the version of the containers. The version +## reported is that of the "binary" and is determined during the ceph build. +## `cephadm version` should NOT require sudo/root. +$CEPHADM_BIN version +$CEPHADM_BIN version | grep 'cephadm version' +# Typically cmake should be running the cephadm build script with CLI arguments +# that embed version info into the "binary". If not using a cephadm build via +# cmake you can set `NO_BUILD_INFO` to skip this check. +if [ -z "$NO_BUILD_INFO" ]; then + $CEPHADM_BIN version | grep -v 'UNSET' + $CEPHADM_BIN version | grep -v 'UNKNOWN' +fi + + +## test shell before bootstrap, when crash dir isn't (yet) present on this host +$CEPHADM shell --fsid $FSID -- ceph -v | grep 'ceph version' +$CEPHADM shell --fsid $FSID -e FOO=BAR -- printenv | grep FOO=BAR + +# test stdin +echo foo | $CEPHADM shell -- cat | grep -q foo + +# the shell commands a bit above this seems to cause the +# /var/lib/ceph/<fsid> directory to be made. Since we now +# check in bootstrap that there are no clusters with the same +# fsid based on the directory existing, we need to make sure +# this directory is gone before bootstrapping. We can +# accomplish this with another rm-cluster +$CEPHADM rm-cluster --fsid $FSID --force + +## bootstrap +ORIG_CONFIG=`mktemp -p $TMPDIR` +CONFIG=`mktemp -p $TMPDIR` +MONCONFIG=`mktemp -p $TMPDIR` +KEYRING=`mktemp -p $TMPDIR` +IP=127.0.0.1 +cat <<EOF > $ORIG_CONFIG +[global] + log to file = true + osd crush chooseleaf type = 0 +EOF +$CEPHADM bootstrap \ + --mon-id a \ + --mgr-id x \ + --mon-ip $IP \ + --fsid $FSID \ + --config $ORIG_CONFIG \ + --output-config $CONFIG \ + --output-keyring $KEYRING \ + --output-pub-ssh-key $TMPDIR/ceph.pub \ + --allow-overwrite \ + --skip-mon-network \ + --skip-monitoring-stack +test -e $CONFIG +test -e $KEYRING +rm -f $ORIG_CONFIG + +$SUDO test -e /var/log/ceph/$FSID/ceph-mon.a.log +$SUDO test -e /var/log/ceph/$FSID/ceph-mgr.x.log + +for u in ceph.target \ + ceph-$FSID.target \ + ceph-$FSID@mon.a \ + ceph-$FSID@mgr.x; do + systemctl is-enabled $u + systemctl is-active $u +done +systemctl | grep system-ceph | grep -q .slice # naming is escaped and annoying + +# check ceph -s works (via shell w/ passed config/keyring) +$CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \ + ceph -s | grep $FSID + +for t in mon mgr node-exporter prometheus grafana; do + $CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \ + ceph orch apply $t --unmanaged +done + +## ls +$CEPHADM ls | jq '.[]' | jq 'select(.name == "mon.a").fsid' \ + | grep $FSID +$CEPHADM ls | jq '.[]' | jq 'select(.name == "mgr.x").fsid' \ + | grep $FSID + +# make sure the version is returned correctly +$CEPHADM ls | jq '.[]' | jq 'select(.name == "mon.a").version' | grep -q \\. + +## deploy +# add mon.b +cp $CONFIG $MONCONFIG +echo "public addrv = [v2:$IP:3301,v1:$IP:6790]" >> $MONCONFIG +jq --null-input \ + --arg fsid $FSID \ + --arg name mon.b \ + --arg keyring /var/lib/ceph/$FSID/mon.a/keyring \ + --arg config "$MONCONFIG" \ + '{"fsid": $fsid, "name": $name, "params":{"keyring": $keyring, "config": $config}}' | \ + $CEPHADM _orch deploy +for u in ceph-$FSID@mon.b; do + systemctl is-enabled $u + systemctl is-active $u +done +cond="$CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \ + ceph mon stat | grep '2 mons'" +is_available "mon.b" "$cond" 30 + +# add mgr.y +$CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \ + ceph auth get-or-create mgr.y \ + mon 'allow profile mgr' \ + osd 'allow *' \ + mds 'allow *' > $TMPDIR/keyring.mgr.y +jq --null-input \ + --arg fsid $FSID \ + --arg name mgr.y \ + --arg keyring $TMPDIR/keyring.mgr.y \ + --arg config "$CONFIG" \ + '{"fsid": $fsid, "name": $name, "params":{"keyring": $keyring, "config": $config}}' | \ + $CEPHADM _orch deploy +for u in ceph-$FSID@mgr.y; do + systemctl is-enabled $u + systemctl is-active $u +done + +for f in `seq 1 30`; do + if $CEPHADM shell --fsid $FSID \ + --config $CONFIG --keyring $KEYRING -- \ + ceph -s -f json-pretty \ + | jq '.mgrmap.num_standbys' | grep -q 1 ; then break; fi + sleep 1 +done +$CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \ + ceph -s -f json-pretty \ + | jq '.mgrmap.num_standbys' | grep -q 1 + +# add osd.{1,2,..} +dd if=/dev/zero of=$TMPDIR/$OSD_IMAGE_NAME bs=1 count=0 seek=$OSD_IMAGE_SIZE +loop_dev=$($SUDO losetup -f) +$SUDO vgremove -f $OSD_VG_NAME || true +$SUDO losetup $loop_dev $TMPDIR/$OSD_IMAGE_NAME +$SUDO pvcreate $loop_dev && $SUDO vgcreate $OSD_VG_NAME $loop_dev + +# osd bootstrap keyring +$CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \ + ceph auth get client.bootstrap-osd > $TMPDIR/keyring.bootstrap.osd + +# create lvs first so ceph-volume doesn't overlap with lv creation +for id in `seq 0 $((--OSD_TO_CREATE))`; do + $SUDO lvcreate -l $((100/$OSD_TO_CREATE))%VG -n $OSD_LV_NAME.$id $OSD_VG_NAME +done + +for id in `seq 0 $((--OSD_TO_CREATE))`; do + device_name=/dev/$OSD_VG_NAME/$OSD_LV_NAME.$id + CEPH_VOLUME="$CEPHADM ceph-volume \ + --fsid $FSID \ + --config $CONFIG \ + --keyring $TMPDIR/keyring.bootstrap.osd --" + + # prepare the osd + $CEPH_VOLUME lvm prepare --bluestore --data $device_name --no-systemd + $CEPH_VOLUME lvm batch --no-auto $device_name --yes --no-systemd + + # osd id and osd fsid + $CEPH_VOLUME lvm list --format json $device_name > $TMPDIR/osd.map + osd_id=$($SUDO cat $TMPDIR/osd.map | jq -cr '.. | ."ceph.osd_id"? | select(.)') + osd_fsid=$($SUDO cat $TMPDIR/osd.map | jq -cr '.. | ."ceph.osd_fsid"? | select(.)') + + # deploy the osd + jq --null-input \ + --arg fsid $FSID \ + --arg name osd.$osd_id \ + --arg keyring $TMPDIR/keyring.bootstrap.osd \ + --arg config "$CONFIG" \ + --arg osd_fsid $osd_fsid \ + '{"fsid": $fsid, "name": $name, "params":{"keyring": $keyring, "config": $config, "osd_fsid": $osd_fsid}}' | \ + $CEPHADM _orch deploy +done + +# add node-exporter +jq --null-input \ + --arg fsid $FSID \ + --arg name node-exporter.a \ + '{"fsid": $fsid, "name": $name}' | \ + ${CEPHADM//--image $IMAGE_DEFAULT/} _orch deploy +cond="curl 'http://localhost:9100' | grep -q 'Node Exporter'" +is_available "node-exporter" "$cond" 10 + +# add prometheus +jq --null-input \ + --arg fsid $FSID \ + --arg name prometheus.a \ + --argjson config_blobs "$(cat ${CEPHADM_SAMPLES_DIR}/prometheus.json)" \ + '{"fsid": $fsid, "name": $name, "config_blobs": $config_blobs}' | \ + ${CEPHADM//--image $IMAGE_DEFAULT/} _orch deploy +cond="curl 'localhost:9095/api/v1/query?query=up'" +is_available "prometheus" "$cond" 10 + +# add grafana +jq --null-input \ + --arg fsid $FSID \ + --arg name grafana.a \ + --argjson config_blobs "$(cat ${CEPHADM_SAMPLES_DIR}/grafana.json)" \ + '{"fsid": $fsid, "name": $name, "config_blobs": $config_blobs}' | \ + ${CEPHADM//--image $IMAGE_DEFAULT/} _orch deploy +cond="curl --insecure 'https://localhost:3000' | grep -q 'grafana'" +is_available "grafana" "$cond" 50 + +# add nfs-ganesha +nfs_stop +nfs_rados_pool=$(cat ${CEPHADM_SAMPLES_DIR}/nfs.json | jq -r '.["pool"]') +$CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \ + ceph osd pool create $nfs_rados_pool 64 +$CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \ + rados --pool nfs-ganesha --namespace nfs-ns create conf-nfs.a +$CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \ + ceph orch pause +jq --null-input \ + --arg fsid $FSID \ + --arg name nfs.a \ + --arg keyring "$KEYRING" \ + --arg config "$CONFIG" \ + --argjson config_blobs "$(cat ${CEPHADM_SAMPLES_DIR}/nfs.json)" \ + '{"fsid": $fsid, "name": $name, "params": {"keyring": $keyring, "config": $config}, "config_blobs": $config_blobs}' | \ + ${CEPHADM} _orch deploy +cond="$SUDO ss -tlnp '( sport = :nfs )' | grep 'ganesha.nfsd'" +is_available "nfs" "$cond" 10 +$CEPHADM shell --fsid $FSID --config $CONFIG --keyring $KEYRING -- \ + ceph orch resume + +# add alertmanager via custom container +alertmanager_image=$(cat ${CEPHADM_SAMPLES_DIR}/custom_container.json | jq -r '.image') +tcp_ports=$(jq .ports ${CEPHADM_SAMPLES_DIR}/custom_container.json) +jq --null-input \ + --arg fsid $FSID \ + --arg name container.alertmanager.a \ + --arg keyring $TMPDIR/keyring.bootstrap.osd \ + --arg config "$CONFIG" \ + --arg image "$alertmanager_image" \ + --argjson tcp_ports "${tcp_ports}" \ + --argjson config_blobs "$(cat ${CEPHADM_SAMPLES_DIR}/custom_container.json)" \ + '{"fsid": $fsid, "name": $name, "image": $image, "params": {"keyring": $keyring, "config": $config, "tcp_ports": $tcp_ports}, "config_blobs": $config_blobs}' | \ + ${CEPHADM//--image $IMAGE_DEFAULT/} _orch deploy +cond="$CEPHADM enter --fsid $FSID --name container.alertmanager.a -- test -f \ + /etc/alertmanager/alertmanager.yml" +is_available "alertmanager.yml" "$cond" 10 +cond="curl 'http://localhost:9093' | grep -q 'Alertmanager'" +is_available "alertmanager" "$cond" 10 + +## run +# WRITE ME + +## unit +$CEPHADM unit --fsid $FSID --name mon.a -- is-enabled +$CEPHADM unit --fsid $FSID --name mon.a -- is-active +expect_false $CEPHADM unit --fsid $FSID --name mon.xyz -- is-active +$CEPHADM unit --fsid $FSID --name mon.a -- disable +expect_false $CEPHADM unit --fsid $FSID --name mon.a -- is-enabled +$CEPHADM unit --fsid $FSID --name mon.a -- enable +$CEPHADM unit --fsid $FSID --name mon.a -- is-enabled +$CEPHADM unit --fsid $FSID --name mon.a -- status +$CEPHADM unit --fsid $FSID --name mon.a -- stop +expect_return_code 3 $CEPHADM unit --fsid $FSID --name mon.a -- status +$CEPHADM unit --fsid $FSID --name mon.a -- start + +## shell +$CEPHADM shell --fsid $FSID -- true +$CEPHADM shell --fsid $FSID -- test -d /var/log/ceph +expect_false $CEPHADM --timeout 10 shell --fsid $FSID -- sleep 60 +$CEPHADM --timeout 60 shell --fsid $FSID -- sleep 10 +$CEPHADM shell --fsid $FSID --mount $TMPDIR $TMPDIR_TEST_MULTIPLE_MOUNTS -- stat /mnt/$(basename $TMPDIR) + +## enter +expect_false $CEPHADM enter +$CEPHADM enter --fsid $FSID --name mon.a -- test -d /var/lib/ceph/mon/ceph-a +$CEPHADM enter --fsid $FSID --name mgr.x -- test -d /var/lib/ceph/mgr/ceph-x +$CEPHADM enter --fsid $FSID --name mon.a -- pidof ceph-mon +expect_false $CEPHADM enter --fsid $FSID --name mgr.x -- pidof ceph-mon +$CEPHADM enter --fsid $FSID --name mgr.x -- pidof ceph-mgr +# this triggers a bug in older versions of podman, including 18.04's 1.6.2 +#expect_false $CEPHADM --timeout 5 enter --fsid $FSID --name mon.a -- sleep 30 +$CEPHADM --timeout 60 enter --fsid $FSID --name mon.a -- sleep 10 + +## ceph-volume +$CEPHADM ceph-volume --fsid $FSID -- inventory --format=json \ + | jq '.[]' + +## preserve test state +[ $CLEANUP = false ] && exit 0 + +## rm-daemon +# mon and osd require --force +expect_false $CEPHADM rm-daemon --fsid $FSID --name mon.a +# mgr does not +$CEPHADM rm-daemon --fsid $FSID --name mgr.x + +expect_false $CEPHADM zap-osds --fsid $FSID +$CEPHADM zap-osds --fsid $FSID --force + +## rm-cluster +expect_false $CEPHADM rm-cluster --fsid $FSID --zap-osds +$CEPHADM rm-cluster --fsid $FSID --force --zap-osds + +echo PASS diff --git a/qa/workunits/cephadm/test_dashboard_e2e.sh b/qa/workunits/cephadm/test_dashboard_e2e.sh new file mode 100755 index 000000000..32e0bcc77 --- /dev/null +++ b/qa/workunits/cephadm/test_dashboard_e2e.sh @@ -0,0 +1,107 @@ +#!/bin/bash -ex + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +DASHBOARD_FRONTEND_DIR=${SCRIPT_DIR}/../../../src/pybind/mgr/dashboard/frontend + +[ -z "$SUDO" ] && SUDO=sudo + +install_common () { + NODEJS_VERSION="16" + if grep -q debian /etc/*-release; then + $SUDO apt-get update + # https://github.com/nodesource/distributions#manual-installation + $SUDO apt-get install curl gpg + KEYRING=/usr/share/keyrings/nodesource.gpg + curl -fsSL https://deb.nodesource.com/gpgkey/nodesource.gpg.key | gpg --dearmor | $SUDO tee "$KEYRING" >/dev/null + DISTRO="$(source /etc/lsb-release; echo $DISTRIB_CODENAME)" + VERSION="node_$NODEJS_VERSION.x" + echo "deb [signed-by=$KEYRING] https://deb.nodesource.com/$VERSION $DISTRO main" | $SUDO tee /etc/apt/sources.list.d/nodesource.list + echo "deb-src [signed-by=$KEYRING] https://deb.nodesource.com/$VERSION $DISTRO main" | $SUDO tee -a /etc/apt/sources.list.d/nodesource.list + $SUDO apt-get update + $SUDO apt-get install nodejs + elif grep -q rhel /etc/*-release; then + $SUDO yum module -y enable nodejs:$NODEJS_VERSION + $SUDO yum install -y jq npm + else + echo "Unsupported distribution." + exit 1 + fi +} + +install_chrome () { + if grep -q debian /etc/*-release; then + $SUDO bash -c 'echo "deb [arch=amd64] https://dl.google.com/linux/chrome/deb/ stable main" > /etc/apt/sources.list.d/google-chrome.list' + curl -fsSL https://dl.google.com/linux/linux_signing_key.pub | $SUDO apt-key add - + $SUDO apt-get update + $SUDO apt-get install -y google-chrome-stable + $SUDO apt-get install -y xvfb + $SUDO rm /etc/apt/sources.list.d/google-chrome.list + elif grep -q rhel /etc/*-release; then + $SUDO dd of=/etc/yum.repos.d/google-chrome.repo status=none <<EOF +[google-chrome] +name=google-chrome +baseurl=https://dl.google.com/linux/chrome/rpm/stable/\$basearch +enabled=1 +gpgcheck=1 +gpgkey=https://dl-ssl.google.com/linux/linux_signing_key.pub +EOF + $SUDO yum install -y google-chrome-stable + $SUDO rm /etc/yum.repos.d/google-chrome.repo + # Cypress dependencies + $SUDO yum install -y xorg-x11-server-Xvfb gtk2-devel gtk3-devel libnotify-devel GConf2 nss.x86_64 libXScrnSaver alsa-lib + else + echo "Unsupported distribution." + exit 1 + fi +} + +cypress_run () { + local specs="$1" + local timeout="$2" + local override_config="excludeSpecPattern=*.po.ts,retries=0,specPattern=${specs}" + + if [ x"$timeout" != "x" ]; then + override_config="${override_config},defaultCommandTimeout=${timeout}" + fi + npx cypress run --browser chrome --headless --config "$override_config" +} + +install_common +install_chrome + +CYPRESS_BASE_URL=$(ceph mgr services | jq -r .dashboard) +export CYPRESS_BASE_URL + +cd $DASHBOARD_FRONTEND_DIR + +# This is required for Cypress to understand typescript +npm ci --unsafe-perm +npx cypress verify +npx cypress info + +# Take `orch device ls` and `orch ps` as ground truth. +ceph orch device ls --refresh +ceph orch ps --refresh +sleep 10 # the previous call is asynchronous +ceph orch device ls --format=json | tee cypress/fixtures/orchestrator/inventory.json +ceph orch ps --format=json | tee cypress/fixtures/orchestrator/services.json + +DASHBOARD_ADMIN_SECRET_FILE="/tmp/dashboard-admin-secret.txt" +printf 'admin' > "${DASHBOARD_ADMIN_SECRET_FILE}" +ceph dashboard ac-user-set-password admin -i "${DASHBOARD_ADMIN_SECRET_FILE}" --force-password + +# Run Dashboard e2e tests. +# These tests are designed with execution order in mind, since orchestrator operations +# are likely to change cluster state, we can't just run tests in arbitrarily order. +# See /ceph/src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/ folder. +find cypress # List all specs + +cypress_run "cypress/e2e/orchestrator/01-hosts.e2e-spec.ts" + +# Hosts are removed and added in the previous step. Do a refresh again. +ceph orch device ls --refresh +sleep 10 +ceph orch device ls --format=json | tee cypress/fixtures/orchestrator/inventory.json + +cypress_run "cypress/e2e/orchestrator/03-inventory.e2e-spec.ts" +cypress_run "cypress/e2e/orchestrator/04-osds.e2e-spec.ts" 300000 diff --git a/qa/workunits/cephadm/test_iscsi_etc_hosts.sh b/qa/workunits/cephadm/test_iscsi_etc_hosts.sh new file mode 100755 index 000000000..adbc34a92 --- /dev/null +++ b/qa/workunits/cephadm/test_iscsi_etc_hosts.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +# checks if the container and host's /etc/hosts files match +# Necessary to avoid potential bugs caused by podman making +# edits to /etc/hosts file in the container +# exits with code 1 if host and iscsi container /etc/hosts do no match + +set -ex + +ISCSI_DAEMON=$(sudo /home/ubuntu/cephtest/cephadm ls | jq -r '.[] | select(.service_name == "iscsi.foo") | .name') +sudo /home/ubuntu/cephtest/cephadm enter --name $ISCSI_DAEMON -- cat /etc/hosts > iscsi_daemon_etc_hosts.txt +if cmp --silent /etc/hosts iscsi_daemon_etc_hosts.txt; then + echo "Daemon and host /etc/hosts files successfully matched" +else + echo "ERROR: /etc/hosts on host did not match /etc/hosts in the iscsi container!" + echo "Host /etc/hosts:" + cat /etc/hosts + echo "Iscsi container /etc/hosts:" + cat iscsi_daemon_etc_hosts.txt + exit 1 +fi diff --git a/qa/workunits/cephadm/test_iscsi_pids_limit.sh b/qa/workunits/cephadm/test_iscsi_pids_limit.sh new file mode 100755 index 000000000..bed4cc9e2 --- /dev/null +++ b/qa/workunits/cephadm/test_iscsi_pids_limit.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# checks if the containers default pids-limit (4096) is removed and Iscsi +# containers continue to run +# exits 1 if fails + +set -ex + +ISCSI_CONT_IDS=$(sudo podman ps -qa --filter='name=iscsi') +CONT_COUNT=$(echo ${ISCSI_CONT_IDS} | wc -w) +test ${CONT_COUNT} -eq 2 + +for i in ${ISCSI_CONT_IDS} +do + test $(sudo podman exec ${i} cat /sys/fs/cgroup/pids/pids.max) == max +done + +for i in ${ISCSI_CONT_IDS} +do + sudo podman exec ${i} /bin/sh -c 'for j in {0..20000}; do sleep 300 & done' +done + +for i in ${ISCSI_CONT_IDS} +do + SLEEP_COUNT=$(sudo podman exec ${i} /bin/sh -c 'ps -ef | grep -c sleep') + test ${SLEEP_COUNT} -gt 20000 +done + +echo OK diff --git a/qa/workunits/cephadm/test_repos.sh b/qa/workunits/cephadm/test_repos.sh new file mode 100755 index 000000000..221585fd0 --- /dev/null +++ b/qa/workunits/cephadm/test_repos.sh @@ -0,0 +1,45 @@ +#!/bin/bash -ex + +SCRIPT_NAME=$(basename ${BASH_SOURCE[0]}) +SCRIPT_DIR=$(dirname ${BASH_SOURCE[0]}) +CEPHADM_SRC_DIR=${SCRIPT_DIR}/../../../src/cephadm + +[ -d "$TMPDIR" ] || TMPDIR=$(mktemp -d tmp.$SCRIPT_NAME.XXXXXX) +trap "$SUDO rm -rf $TMPDIR" EXIT + +if [ -z "$CEPHADM" ]; then + CEPHADM=`mktemp -p $TMPDIR tmp.cephadm.XXXXXX` + ${CEPHADM_SRC_DIR}/build.sh "$CEPHADM" +fi + +# this is a pretty weak test, unfortunately, since the +# package may also be in the base OS. +function test_install_uninstall() { + ( sudo apt update && \ + sudo apt -y install cephadm && \ + sudo $CEPHADM install && \ + sudo apt -y remove cephadm ) || \ + ( sudo yum -y install cephadm && \ + sudo $CEPHADM install && \ + sudo yum -y remove cephadm ) || \ + ( sudo dnf -y install cephadm && \ + sudo $CEPHADM install && \ + sudo dnf -y remove cephadm ) || \ + ( sudo zypper -n install cephadm && \ + sudo $CEPHADM install && \ + sudo zypper -n remove cephadm ) +} + +sudo $CEPHADM -v add-repo --release octopus +test_install_uninstall +sudo $CEPHADM -v rm-repo + +sudo $CEPHADM -v add-repo --dev main +test_install_uninstall +sudo $CEPHADM -v rm-repo + +sudo $CEPHADM -v add-repo --release 15.2.7 +test_install_uninstall +sudo $CEPHADM -v rm-repo + +echo OK. |