diff options
Diffstat (limited to 'qa/cephfs/unshare_ns_mount.sh')
| -rwxr-xr-x | qa/cephfs/unshare_ns_mount.sh | 594 |
1 files changed, 594 insertions, 0 deletions
diff --git a/qa/cephfs/unshare_ns_mount.sh b/qa/cephfs/unshare_ns_mount.sh new file mode 100755 index 000000000..88ac3e933 --- /dev/null +++ b/qa/cephfs/unshare_ns_mount.sh @@ -0,0 +1,594 @@ +#!/usr/bin/env bash + +# This is one helper for mounting the ceph-fuse/kernel clients by +# unsharing the network namespace, let's call it netns container. +# With the netns container, you can easily suspend or resume the +# virtual network interface to simulate the client node hard +# shutdown for some test cases. +# +# netnsX netnsY netnsZ +# -------------- -------------- -------------- +# | mount client | | mount client | | mount client | +# | default | ... | default | ... | default | +# |192.168.0.1/16| |192.168.0.2/16| |192.168.0.3/16| +# | veth0 | | veth0 | | veth0 | +# -------------- -------------- ------------- +# | | | +# \ | brx.Y / +# \ ---------------------- / +# \ brx.X | ceph-brx | brx.Z / +# \------>| default |<------/ +# | | 192.168.255.254/16 | | +# | ---------------------- | +# (suspend/resume) | (suspend/resume) +# ----------- +# | Physical | +# | A.B.C.D/M | +# ----------- +# +# Defaultly it will use the 192.168.X.Y/16 private network IPs for +# the ceph-brx and netnses as above. And you can also specify your +# own new ip/mask for the ceph-brx, like: +# +# $ unshare_ns_mount.sh --fuse /mnt/cephfs --brxip 172.19.100.100/12 +# +# Then the each netns will get a new ip from the ranges: +# [172.16.0.1 ~ 172.19.100.99]/12 and [172.19.100.101 ~ 172.31.255.254]/12 + +usage() { + echo "" + echo "This will help to isolate the network namespace from OS for the mount client!" + echo "" + echo "usage: unshare_ns_mount.sh [OPTIONS [paramters]] [--brxip <ip_address/mask>]" + echo "OPTIONS:" + echo -e " --fuse <ceph-fuse options>" + echo -e "\tThe ceph-fuse command options" + echo -e "\t $ unshare_ns_mount.sh --fuse -m 192.168.0.1:6789 /mnt/cephfs -o nonempty" + echo "" + echo -e " --kernel <mount options>" + echo -e "\tThe mount command options" + echo -e "\t $ unshare_ns_mount.sh --kernel -t ceph 192.168.0.1:6789:/ /mnt/cephfs -o fs=a" + echo "" + echo -e " --suspend <mountpoint>" + echo -e "\tDown the veth interface in the network namespace" + echo -e "\t $ unshare_ns_mount.sh --suspend /mnt/cephfs" + echo "" + echo -e " --resume <mountpoint>" + echo -e "\tUp the veth interface in the network namespace" + echo -e "\t $ unshare_ns_mount.sh --resume /mnt/cephfs" + echo "" + echo -e " --umount <mountpoint>" + echo -e "\tUmount and delete the network namespace" + echo -e "\t $ unshare_ns_mount.sh --umount /mnt/cephfs" + echo "" + echo -e " --brxip <ip_address/mask>" + echo -e "\tSpecify ip/mask for ceph-brx and it only makes sense for --fuse/--kernel options" + echo -e "\t(default: 192.168.255.254/16, netns ip: 192.168.0.1/16 ~ 192.168.255.253/16)" + echo -e "\t $ unshare_ns_mount.sh --fuse -m 192.168.0.1:6789 /mnt/cephfs --brxip 172.19.255.254/12" + echo -e "\t $ unshare_ns_mount.sh --kernel 192.168.0.1:6789:/ /mnt/cephfs --brxip 172.19.255.254/12" + echo "" + echo -e " -h, --help" + echo -e "\tPrint help" + echo "" +} + +CEPH_BRX=ceph-brx +CEPH_BRX_IP_DEF=192.168.255.254 +NET_MASK_DEF=16 +BRD_DEF=192.168.255.255 + +CEPH_BRX_IP=$CEPH_BRX_IP_DEF +NET_MASK=$NET_MASK_DEF +BRD=$BRD_DEF + +mountpoint="" +new_netns="" +fuse_type=false + +function get_mountpoint() { + for param in $@ + do + if [ -d $param ]; then + # skipping "--client_mountpoint/-r root_directory" + # option for ceph-fuse command + if [ "$last" == "-r" -o "$last" == "--client_mountpoint" ]; then + last=$param + continue + fi + if [ "0$mountpoint" != "0" ]; then + echo "Oops: too many mountpiont options!" + exit 1 + fi + mountpoint=$param + fi + last=$param + done + + if [ "0$mountpoint" == "0" ]; then + echo "Oops: mountpoint path is not a directory or no mountpoint specified!" + exit 1 + fi +} + +function get_new_netns() { + # prune the repeating slashes: + # "/mnt///cephfs///" --> "/mnt/cephfs/" + __mountpoint=`echo "$mountpoint" | sed 's/\/\+/\//g'` + + # prune the leading slashes + while [ ${__mountpoint:0:1} == "/" ] + do + __mountpoint=${__mountpoint:1} + done + + # prune the last slashes + while [ ${__mountpoint: -1} == "/" ] + do + __mountpoint=${__mountpoint:0:-1} + done + + # replace '/' with '-' + __mountpoint=${__mountpoint//\//-} + + # "mnt/cephfs" --> "ceph-fuse-mnt-cephfs" + if [ "$1" == "--fuse" ]; then + new_netns=`echo ceph-fuse-$__mountpoint` + fuse_type=true + return + fi + + # "mnt/cephfs" --> "ceph-kernel-mnt-cephfs" + if [ "$1" == "--kernel" ]; then + new_netns=`echo ceph-kernel-$__mountpoint` + return + fi + + # we are in umount/suspend/resume routines + for ns in `ip netns list | awk '{print $1}'` + do + if [ "$ns" == "ceph-fuse-$__mountpoint" ]; then + new_netns=$ns + fuse_type=true + return + fi + if [ "$ns" == "ceph-kernel-$__mountpoint" ]; then + new_netns=$ns + return + fi + done + + if [ "0$new_netns" == "0" ]; then + echo "Oops, netns 'ceph-{fuse/kernel}-$__mountpoint' does not exists!" + exit 1 + fi +} + +# the peer veth name will be "brx.$nsid" on host node +function get_netns_brx() { + get_new_netns + + nsid=`ip netns list-id | grep "$new_netns" | awk '{print $2}'` + netns_veth=brx.$nsid + eval $1="$netns_veth" +} + +function suspend_netns_veth() { + get_mountpoint $@ + + get_netns_brx brx + ip link set $brx down + exit 0 +} + +function resume_netns_veth() { + get_mountpoint $@ + + get_netns_brx brx + ip link set $brx up + exit 0 +} + +# help and usage +if [ $# == 0 -o "$1" == "-h" -o "$1" == "--help" ]; then + usage + exit 0 +fi + +# suspend the veth from network namespace +if [ $1 == "--suspend" ]; then + suspend_netns_veth $@ + exit 0 +fi + +# resume the veth from network namespace +if [ $1 == "--resume" ]; then + resume_netns_veth $@ + exit 0 +fi + +function ceph_umount() { + get_mountpoint $@ + get_new_netns + + if [ $fuse_type == true ]; then + nsenter --net=/var/run/netns/$new_netns fusermount -u $mountpoint 2>/dev/null + else + nsenter --net=/var/run/netns/$new_netns umount $mountpoint 2>/dev/null + fi + + # let's wait for a while to let the umount operation + # to finish before deleting the netns + while [ 1 ] + do + for pid in `ip netns pids $new_netns 2>/dev/null` + do + name=`cat /proc/$pid/comm 2>/dev/null` + if [ "$name" == "ceph-fuse" ]; then + break + fi + done + + if [ "$name" == "ceph-fuse" ]; then + name="" + usleep 100000 + continue + fi + + break + done + + nsid=`ip netns list-id | grep "$new_netns" | awk '{print $2}'` + netns_brx=brx.$nsid + + # brctl delif $CEPH_BRX $netns_brx 2>/dev/null + nmcli connection down $netns_brx down 2>/dev/null + nmcli connection delete $netns_brx 2>/dev/null + + ip netns delete $new_netns 2>/dev/null + + # if this is the last netns_brx, will delete + # the $CEPH_BRX and restore the OS configure + # rc=`brctl show ceph-brx 2>/dev/null | grep 'brx\.'|wc -l` + rc=`nmcli connection show 2>/dev/null | grep 'brx\.' | wc -l` + if [ $rc == 0 ]; then + ip link set $CEPH_BRX down 2>/dev/null + # brctl delbr $CEPH_BRX 2>/dev/null + nmcli connection delete $CEPH_BRX 2>/dev/null + + # restore the ip forward + tmpfile=`ls /tmp/ | grep "$CEPH_BRX\."` + tmpfile=/tmp/$tmpfile + if [ ! -f $tmpfile ]; then + echo "Oops, the $CEPH_BRX.XXX temp file does not exist!" + else + save=`cat $tmpfile` + echo $save > /proc/sys/net/ipv4/ip_forward + rm -rf $tmpfile + fi + + # drop the iptables NAT rules + host_nic=`route | grep default | awk '{print $8}'` + iptables -D FORWARD -o $host_nic -i $CEPH_BRX -j ACCEPT + iptables -D FORWARD -i $host_nic -o $CEPH_BRX -j ACCEPT + iptables -t nat -D POSTROUTING -s $CEPH_BRX_IP/$NET_MASK -o $host_nic -j MASQUERADE + fi +} + +function get_brd_mask() { + first=`echo "$CEPH_BRX_IP" | awk -F. '{print $1}'` + second=`echo "$CEPH_BRX_IP" | awk -F. '{print $2}'` + third=`echo "$CEPH_BRX_IP" | awk -F. '{print $3}'` + fourth=`echo "$CEPH_BRX_IP" | awk -F. '{print $4}'` + + if [ "$first" == "172" ]; then + second_max=31 + else + second_max=255 + fi + third_max=255 + fourth_max=255 + + if [ $NET_MASK -lt 16 ]; then + let power=16-$NET_MASK + m=`awk 'BEGIN{printf 2^"'$power'"-1}'` + second=$((second&~m)) + let second_max=$second+$m + elif [ $NET_MASK -lt 24 ]; then + let power=24-$NET_MASK + m=`awk 'BEGIN{printf 2^"'$power'"-1}'` + third=$((third&~m)) + let third_max=$third+$m + second_max=$second + elif [ $NET_MASK -lt 32 ]; then + let power=32-$NET_MASK + m=`awk 'BEGIN{printf 2^"'$power'"-1}'` + fourth=$((fourth&~m)) + let fourth_max=$fourth+$m + second_max=$second + third_max=$third + fi + + BRD=$first.$second_max.$third_max.$fourth_max +} + +# As default: +# The netns IP will be 192.168.0.1 ~ 192.168.255.253, +# and 192.168.255.254 is saved for $CEPH_BRX +function get_new_ns_ip() { + first=`echo "$CEPH_BRX_IP" | awk -F. '{print $1}'` + second=`echo "$CEPH_BRX_IP" | awk -F. '{print $2}'` + third=`echo "$CEPH_BRX_IP" | awk -F. '{print $3}'` + fourth=`echo "$CEPH_BRX_IP" | awk -F. '{print $4}'` + + if [ "$first" == ""172 ]; then + second_max=31 + else + second_max=255 + fi + third_max=255 + fourth_max=254 + + if [ $NET_MASK -lt 16 ]; then + let power=16-$NET_MASK + m=`awk 'BEGIN{printf 2^"'$power'"-1}'` + second=$((second&~m)) + let second_max=$second+$m + third=0 + fourth=1 + elif [ $NET_MASK -lt 24 ]; then + let power=24-$NET_MASK + m=`awk 'BEGIN{printf 2^"'$power'"-1}'` + third=$((third&~m)) + let third_max=$third+$m + second_max=$second + fourth=1 + elif [ $NET_MASK -lt 32 ]; then + let power=32-$NET_MASK + m=`awk 'BEGIN{printf 2^"'$power'"-1}'` + fourth=$((fourth&~m)) + let fourth+=1 + let fourth_max=$fourth+$m-1 + second_max=$second + third_max=$third + fi + + while [ $second -le $second_max -a $third -le $third_max -a $fourth -le $fourth_max ] + do + conflict=false + + # check from the existing network namespaces + for netns in `ip netns list | awk '{print $1}'` + do + ip=`ip netns exec $netns ip addr | grep "inet " | grep "veth0"` + ip=`echo "$ip" | awk '{print $2}' | awk -F/ '{print $1}'` + if [ "0$ip" == "0" ]; then + continue + fi + if [ "$first.$second.$third.$fourth" == "$ip" ]; then + conflict=true + + let fourth+=1 + if [ $fourth -le $fourth_max ]; then + break + fi + + fourth=0 + let third+=1 + if [ $third -le $third_max ]; then + break + fi + + third=0 + let second+=1 + if [ $second -le $second_max ]; then + break + fi + + echo "Oops: we have ran out of the ip addresses!" + exit 1 + fi + done + + # have we found one ? + if [ $conflict == false ]; then + break + fi + done + + ip=$first.$second.$third.$fourth + max=$first.$second_max.$third_max.$fourth_max + if [ "$ip" == "$max" ]; then + echo "Oops: we have ran out of the ip addresses!" + exit 1 + fi + + eval $1="$ip" +} + +function check_valid_private_ip() { + first=`echo "$1" | awk -F. '{print $1}'` + second=`echo "$1" | awk -F. '{print $2}'` + + # private network class A 10.0.0.0 - 10.255.255.255 + if [ "$first" == "10" -a $NET_MASK -ge 8 ]; then + return + fi + + # private network class B 172.16.0.0 - 172.31.255.255 + if [ "$first" == "172" -a $second -ge 16 -a $second -le 31 -a $NET_MASK -ge 12 ]; then + return + fi + + # private network class C 192.168.0.0 - 192.168.255.255 + if [ "$first" == "192" -a "$second" == "168" -a $NET_MASK -ge 16 ]; then + return + fi + + echo "Oops: invalid private ip address '$CEPH_BRX_IP/$NET_MASK'!" + exit 1 +} + +function setup_bridge_and_nat() { + # check and parse the --brxip parameter + is_brxip=false + for ip in $@ + do + if [ "$ip" == "--brxip" ]; then + is_brxip=true + continue + fi + if [ $is_brxip == true ]; then + new_brxip=$ip + break + fi + done + + # if the $CEPH_BRX already exists, then check the new + # brxip, if not match fail it without doing anything. + rc=`ip addr | grep "inet " | grep " $CEPH_BRX"` + if [ "0$rc" != "0" ]; then + existing_brxip=`echo "$rc" | awk '{print $2}'` + if [ "0$new_brxip" != "0" -a "$existing_brxip" != "$new_brxip" ]; then + echo "Oops: conflict with the existing $CEPH_BRX ip '$existing_brxip', new '$new_brxip'!" + exit 1 + fi + + CEPH_BRX_IP=`echo "$existing_brxip" | awk -F/ '{print $1}'` + NET_MASK=`echo "$existing_brxip" | awk -F/ '{print $2}'` + get_brd_mask + return + fi + + # if it is the first time to run the the script or there + # is no any network namespace exists, we need to setup + # the $CEPH_BRX, if no --brxip is specified will use the + # default $CEPH_BRX_IP/$NET_MASK + if [ "0$new_brxip" != "0" ]; then + CEPH_BRX_IP=`echo "$new_brxip" | awk -F/ '{print $1}'` + NET_MASK=`echo "$new_brxip" | awk -F/ '{print $2}'` + get_brd_mask + check_valid_private_ip $CEPH_BRX_IP + fi + + # brctl addbr $CEPH_BRX + nmcli connection add type bridge con-name $CEPH_BRX ifname $CEPH_BRX stp no + # ip link set $CEPH_BRX up + # ip addr add $CEPH_BRX_IP/$NET_MASK brd $BRD dev $CEPH_BRX + nmcli connection modify $CEPH_BRX ipv4.addresses $CEPH_BRX_IP/$NET_MASK ipv4.method manual + nmcli connection up $CEPH_BRX + + # setup the NAT + rm -rf /tmp/ceph-brx.* + tmpfile=$(mktemp /tmp/ceph-brx.XXXXXXXX) + save=`cat /proc/sys/net/ipv4/ip_forward` + echo $save > $tmpfile + echo 1 > /proc/sys/net/ipv4/ip_forward + + host_nic=`route | grep default | awk '{print $8}'` + iptables -A FORWARD -o $host_nic -i $CEPH_BRX -j ACCEPT + iptables -A FORWARD -i $host_nic -o $CEPH_BRX -j ACCEPT + iptables -t nat -A POSTROUTING -s $CEPH_BRX_IP/$NET_MASK -o $host_nic -j MASQUERADE +} + +function __ceph_mount() { + # for some options like the '-t' in mount command + # the nsenter command will take over it, so it is + # hard to pass it direct to the netns. + # here we will create one temp file with x mode + tmpfile=$(mktemp /tmp/ceph-nsenter.XXXXXXXX) + chmod +x $tmpfile + if [ "$1" == "--kernel" ]; then + cmd=`echo "$@" | sed 's/--kernel/mount/'` + else + cmd=`echo "$@" | sed 's/--fuse/ceph-fuse/'` + fi + + # remove the --brxip parameter + cmd=`echo "$cmd" | sed 's/--brxip.*\/[0-9]* //'` + + # enter $new_netns and run ceph fuse client mount, + # we couldn't use 'ip netns exec' here because it + # will unshare the mount namespace. + echo "$cmd" > $tmpfile + nsenter --net=/var/run/netns/$new_netns /bin/bash $tmpfile ; echo $? > $tmpfile + rc=`cat $tmpfile` + rm -f $tmpfile + + # fall back + if [ $rc != 0 ]; then + m=$mountpoint + mountpoint="" + ceph_umount $m + fi +} + +function get_new_nsid() { + # get one uniq netns id + uniq_id=0 + while [ 1 ] + do + rc=`ip netns list-id | grep "nsid $uniq_id "` + if [ "0$rc" == "0" ]; then + break + fi + let uniq_id+=1 + done + + eval $1="$uniq_id" +} + +function ceph_mount() { + get_mountpoint $@ + setup_bridge_and_nat $@ + + get_new_netns $1 + rc=`ip netns list | grep "$new_netns" | awk '{print $1}'` + if [ "0$rc" != "0" ]; then + echo "Oops: the netns "$new_netns" already exists!" + exit 1 + fi + + get_new_nsid new_nsid + + # create a new network namespace + ip netns add $new_netns + ip netns set $new_netns $new_nsid + + get_new_ns_ip ns_ip + if [ 0"$ns_ip" == "0" ]; then + echo "Oops: there is no ip address could be used any more!" + exit 1 + fi + + # veth interface in netns + ns_veth=veth0 + netns_brx=brx.$new_nsid + + # setup veth interfaces + ip link add $ns_veth netns $new_netns type veth peer name $netns_brx + ip netns exec $new_netns ip addr add $ns_ip/$NET_MASK brd $BRD dev $ns_veth + ip netns exec $new_netns ip link set $ns_veth up + ip netns exec $new_netns ip link set lo up + ip netns exec $new_netns ip route add default via $CEPH_BRX_IP + + # bring up the bridge interface and join it to $CEPH_BRX + # brctl addif $CEPH_BRX $netns_brx + nmcli connection add type bridge-slave con-name $netns_brx ifname $netns_brx master $CEPH_BRX + nmcli connection up $netns_brx + # ip link set $netns_brx up + + __ceph_mount $@ +} + +if [ "$1" == "--umount" ]; then + ceph_umount $@ + exit 0 +fi + +# mount in the netns +if [ "$1" != "--kernel" -a "$1" != "--fuse" ]; then + echo "Oops: invalid mount options '$1'!" + exit 1 +fi + +ceph_mount $@ |