summaryrefslogtreecommitdiffstats
path: root/ctdb/tests/CLUSTER/complex/scripts/local.bash
blob: 0ef5c0acae82195fd79ea6ff798b7ac1ef8192f5 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
# Hey Emacs, this is a -*- shell-script -*- !!!  :-)

# Thanks/blame to Stephen Rothwell for suggesting that this can be
# done in the shell.  ;-)
ipv6_to_hex ()
{
    local addr="$1"

    # Replace "::" by something special.
    local foo="${addr/::/:@:}"

    # Join the groups of digits together, 0-padding each group of
    # digits out to 4 digits, and count the number of (non-@) groups
    local out=""
    local count=0
    local i
    for i in $(IFS=":" ; echo $foo ) ; do
	if [ "$i" = "@" ] ; then
	    out="${out}@"
	else
	    out="${out}$(printf '%04x' 0x${i})"
	    count=$(($count + 4))
	fi
    done

    # Replace '@' with correct number of zeroes
    local zeroes=$(printf "%0$((32 - $count))x" 0)
    echo "${out/@/${zeroes}}"
}

#######################################

get_src_socket ()
{
    local proto="$1"
    local dst_socket="$2"
    local pid="$3"
    local prog="$4"

    local pat="^${proto}6?[[:space:]]+[[:digit:]]+[[:space:]]+[[:digit:]]+[[:space:]]+[^[:space:]]+[[:space:]]+${dst_socket//./\\.}[[:space:]]+ESTABLISHED[[:space:]]+${pid}/${prog}[[:space:]]*\$"
    out=$(netstat -tanp |
	grep -E "$pat" |
	awk '{ print $4 }')

    [ -n "$out" ]
}

wait_until_get_src_socket ()
{
    local proto="$1"
    local dst_socket="$2"
    local pid="$3"
    local prog="$4"

    echo "Waiting for ${prog} to establish connection to ${dst_socket}..."

    wait_until 5 get_src_socket "$@"
}

#######################################

check_tickles ()
{
    local node="$1"
    local test_ip="$2"
    local test_port="$3"
    local src_socket="$4"
    try_command_on_node $node ctdb gettickles $test_ip $test_port
    # SRC: 10.0.2.45:49091   DST: 10.0.2.143:445
    grep -Fq "SRC: ${src_socket} " "$outfile"
}

check_tickles_all ()
{
    local numnodes="$1"
    local test_ip="$2"
    local test_port="$3"
    local src_socket="$4"

    try_command_on_node all ctdb gettickles $test_ip $test_port
    # SRC: 10.0.2.45:49091   DST: 10.0.2.143:445
    local count=$(grep -Fc "SRC: ${src_socket} " "$outfile" || true)
    [ $count -eq $numnodes ]
}



#######################################

# filename will be in $tcpdump_filename, pid in $tcpdump_pid
tcpdump_start ()
{
    tcpdump_filter="$1" # global

    echo "Running tcpdump..."
    tcpdump_filename=$(mktemp)
    ctdb_test_exit_hook_add "rm -f $tcpdump_filename"

    # The only way of being sure that tcpdump is listening is to send
    # some packets that it will see.  So we use dummy pings - the -U
    # option to tcpdump ensures that packets are flushed to the file
    # as they are captured.
    local dummy_addr="127.3.2.1"
    local dummy="icmp and dst host ${dummy_addr} and icmp[icmptype] == icmp-echo"
    tcpdump -n -p -s 0 -e -U -w $tcpdump_filename -i any "($tcpdump_filter) or ($dummy)" &
    ctdb_test_exit_hook_add "kill $! >/dev/null 2>&1"

    echo "Waiting for tcpdump output file to be ready..."
    ping -q "$dummy_addr" >/dev/null 2>&1 &
    ctdb_test_exit_hook_add "kill $! >/dev/null 2>&1"

    tcpdump_listen_for_dummy ()
    {
	tcpdump -n -r $tcpdump_filename -c 1 "$dummy" >/dev/null 2>&1
    }

    wait_until 10 tcpdump_listen_for_dummy
}

# By default, wait for 1 matching packet.
tcpdump_wait ()
{
	local count="${1:-1}"
	local filter="${2:-${tcpdump_filter}}"

	tcpdump_check ()
	{
		# It would be much nicer to add "ether src
		# $releasing_mac" to the filter.  However, tcpdump
		# does not allow MAC filtering unless an ethernet
		# interface is specified with -i.  It doesn't work
		# with "-i any" and it doesn't work when reading from
		# a file.  :-(
		local found
		if [ -n "$releasing_mac" ] ; then
			found=$(tcpdump -n -e -r "$tcpdump_filename" \
					"$filter" 2>/dev/null |
					       grep -c "In ${releasing_mac}")
		else
			found=$(tcpdump -n -e -r "$tcpdump_filename" \
					"$filter" 2>/dev/null |
					       wc -l)
		fi

		[ $found -ge $count ]
	}

	echo "Waiting for tcpdump to capture some packets..."
	if ! wait_until 30 tcpdump_check ; then
		echo "DEBUG AT $(date '+%F %T'):"
		local i
		for i in "onnode -q 0 $CTDB status" \
				 "netstat -tanp" \
				 "tcpdump -n -e -r $tcpdump_filename" ; do
			echo "$i"
			$i || true
		done
		return 1
	fi
}

tcpdump_show ()
{
    local filter="${1:-${tcpdump_filter}}"

    tcpdump -n -e -vv -XX -r $tcpdump_filename  "$filter" 2>/dev/null
}

tcp4tickle_sniff_start ()
{
    local src="$1"
    local dst="$2"

    local in="src host ${dst%:*} and tcp src port ${dst##*:} and dst host ${src%:*} and tcp dst port ${src##*:}"
    local out="src host ${src%:*} and tcp src port ${src##*:} and dst host ${dst%:*} and tcp dst port ${dst##*:}"
    local tickle_ack="${in} and (tcp[tcpflags] & tcp-ack != 0) and (tcp[14:2] == 1234)" # win == 1234
    local ack_ack="${out} and (tcp[tcpflags] & tcp-ack != 0)"
    tcptickle_reset="${in} and tcp[tcpflags] & tcp-rst != 0"
    local filter="(${tickle_ack}) or (${ack_ack}) or (${tcptickle_reset})"

    tcpdump_start "$filter"
}

# tcp[] does not work for IPv6 (in some versions of tcpdump)
tcp6tickle_sniff_start ()
{
    local src="$1"
    local dst="$2"

    local in="src host ${dst%:*} and tcp src port ${dst##*:} and dst host ${src%:*} and tcp dst port ${src##*:}"
    local out="src host ${src%:*} and tcp src port ${src##*:} and dst host ${dst%:*} and tcp dst port ${dst##*:}"
    local tickle_ack="${in} and (ip6[53] & tcp-ack != 0) and (ip6[54:2] == 1234)" # win == 1234
    local ack_ack="${out} and (ip6[53] & tcp-ack != 0)"
    tcptickle_reset="${in} and ip6[53] & tcp-rst != 0"
    local filter="(${tickle_ack}) or (${ack_ack}) or (${tcptickle_reset})"

    tcpdump_start "$filter"
}

tcptickle_sniff_start ()
{
    local src="$1"
    local dst="$2"

    case "${dst%:*}" in
	*:*) tcp6tickle_sniff_start "$src" "$dst" ;;
	*)   tcp4tickle_sniff_start "$src" "$dst" ;;
    esac
}

tcptickle_sniff_wait_show ()
{
	local releasing_mac="$1"  # optional, used by tcpdump_wait()

	tcpdump_wait 1 "$tcptickle_reset"

	echo "GOOD: here are some TCP tickle packets:"
	tcpdump_show
}

gratarp4_sniff_start ()
{
    tcpdump_start "arp host ${test_ip}"
}

gratarp6_sniff_start ()
{
    local neighbor_advertisement="icmp6 and ip6[40] == 136"
    local hex=$(ipv6_to_hex "$test_ip")
    local match_target="ip6[48:4] == 0x${hex:0:8} and ip6[52:4] == 0x${hex:8:8} and ip6[56:4] == 0x${hex:16:8} and ip6[60:4] == 0x${hex:24:8}"

    tcpdump_start "${neighbor_advertisement} and ${match_target}"
}

gratarp_sniff_start ()
{
    case "$test_ip" in
	*:*) gratarp6_sniff_start ;;
	*)   gratarp4_sniff_start ;;
    esac
}

gratarp_sniff_wait_show ()
{
    tcpdump_wait 2

    echo "GOOD: this should be the some gratuitous ARPs:"
    tcpdump_show
}

ping_wrapper ()
{
    case "$*" in
	*:*) ping6 "$@"   ;;
	*)   ping  "$@"   ;;
    esac
}

#######################################

nfs_test_setup ()
{
    select_test_node_and_ips

    nfs_first_export=$(showmount -e $test_ip | sed -n -e '2s/ .*//p')

    echo "Creating test subdirectory..."
    try_command_on_node $test_node "TMPDIR=$nfs_first_export mktemp -d"
    nfs_test_dir="$out"
    try_command_on_node $test_node "chmod 777 $nfs_test_dir"

    nfs_mnt_d=$(mktemp -d)
    nfs_local_file="${nfs_mnt_d}/${nfs_test_dir##*/}/TEST_FILE"
    nfs_remote_file="${nfs_test_dir}/TEST_FILE"

    ctdb_test_exit_hook_add nfs_test_cleanup

    echo "Mounting ${test_ip}:${nfs_first_export} on ${nfs_mnt_d} ..."
    mount -o timeo=1,hard,intr,vers=3 \
	"[${test_ip}]:${nfs_first_export}" ${nfs_mnt_d}
}

nfs_test_cleanup ()
{
    rm -f "$nfs_local_file"
    umount -f "$nfs_mnt_d"
    rmdir "$nfs_mnt_d"
    onnode -q $test_node rmdir "$nfs_test_dir"
}