summaryrefslogtreecommitdiffstats
path: root/ctdb/tests/INTEGRATION/database/fetch.002.ring-hotkeys.sh
blob: 6d442535a2125c8ea083dd1b08fd607a09c75f66 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
#!/usr/bin/env bash

# Run the fetch_ring test, sanity check the output and check hot keys
# statistics

. "${TEST_SCRIPTS_DIR}/integration.bash"

set -e

ctdb_test_init

testdb="fetch_ring.tdb"

ctdb_get_all_pnns
# $all_pnns is set above
# shellcheck disable=SC2154
num_nodes=$(echo "$all_pnns" | wc -w | tr -d '[:space:]')
first=$(echo "$all_pnns" | sed -n -e '1p')

get_key ()
{
	_n="$1"

	echo "testkey${_n}"
}

run_fetch_ring ()
{
	_timelimit="$1"
	_key_num="$2"

	_key=$(get_key "$_key_num")
	_base_cmd="fetch_ring -n ${num_nodes} -D ${testdb}"
	_cmd="${_base_cmd} -t ${_timelimit} -k ${_key}"
	echo "Running \"${_cmd}\" on all $num_nodes nodes."
	testprog_onnode -v -p all "$_cmd"

	_pat='^(Waiting for cluster|Fetch\[[[:digit:]]+\]: [[:digit:]]+(\.[[:digit:]]+)? msgs/sec)$'
	sanity_check_output 1 "$_pat"

	# Get the last line of output.
	# $outfile is set above by testprog_onnode()
	# shellcheck disable=SC2154
	_last=$(tail -n 1 "$outfile")

	# $last should look like this:
	#    Fetch[1]: 10670.93 msgs/sec
	_stuff="${_last##*Fetch\[*\]: }"
	_mps="${_stuff% msgs/sec*}"

	if [ "${_mps%.*}" -ge 10 ] ; then
		echo "OK: ${_mps} msgs/sec >= 10 msgs/sec"
	else
		ctdb_test_fail "BAD: ${_mps} msgs/sec < 10 msgs/sec"
	fi
}

check_hot_keys ()
{
	_pnn="$1"
	_first_key="$2"
	_num_keys="$3"

	echo
	echo "Checking hot keys on node ${_pnn}"

	ctdb_onnode "$_pnn" dbstatistics "$testdb"

	# Get hot keys with a non-empty key
	_hotkeys=$(grep -Ex '[[:space:]]+Count:[[:digit:]]+ Key:[[:xdigit:]]+' \
			"$outfile") || true

	# Check that there are the right number of non-empty slots
	if [ -z "$_hotkeys" ] ; then
		_num=0
	else
		_num=$(echo "$_hotkeys" | wc -l | tr -d '[:space:]')
	fi
	_msg="hot key slots in use = ${_num}"
	if [ "$_num_keys" -ne "$_num" ] ; then
		echo
		cat "$outfile"
		ctdb_test_fail "BAD: ${_msg} (expected ${_num_keys})"
	fi
	echo "GOOD: ${_msg}"

	# No hot keys?  Done...
	if [ "$_num" = 0 ] ; then
		return
	fi

	# Check that hot key counts are correctly sorted
	#
	# Try to be as POSIX as possible
	# shellcheck disable=SC2001
	_counts=$(echo "$_hotkeys" | \
			  sed -e 's|.*Count:\([[:digit:]][[:digit:]]*\).*|\1|')
	_counts_sorted=$(echo "$_counts" | sort -n)
	if [ "$_counts" != "$_counts_sorted" ] ; then
		echo
		cat "$outfile"
		ctdb_test_fail "BAD: hot keys not sorted"
	fi
	echo "GOOD: hot key counts are correctly sorted"

	# Check that all keys are considered hot
	for _j in $(seq "$_first_key" $((_first_key + _num_keys - 1))) ; do
		_key=$(get_key "$_j")
		_key_hex=$(printf '%s' "$_key" | \
				   od -A n -t x1 | \
				   tr -d '[:space:]')
		if ! echo "$_hotkeys" | grep -q "Key:${_key_hex}\$" ; then
			echo
			cat "$outfile"
			ctdb_test_fail "BAD: key \"${_key}\" is not a hot key"
		fi
	done
	echo "GOOD: all keys are listed as hot keys"
}

# Run fetch_ring for each of 10 keys.  After each run confirm that all
# keys used so far are considered hot keys (and do other hot key
# sanity checks) on all nodes.
for i in $(seq 1 10) ; do
	run_fetch_ring 5 "$i"

	for pnn in $all_pnns ; do
		check_hot_keys "$pnn" 1 "$i"
	done

	echo
done

echo
echo "Resetting statistics on node ${first}"
ctdb_onnode "$first" statisticsreset

# Ensure that only node $first has had statistics reset
for pnn in $all_pnns ; do
	if [ "$pnn" = "$first" ] ; then
		check_hot_keys "$pnn" 1 0
	else
		check_hot_keys "$pnn" 1 10
	fi
done

echo

# Run fetch_ring for each of 3 new keys.  After each run confirm that
# the new keys used so far are considered hot keys (and do other hot
# key sanity checks) on node $first.
#
# Note that nothing can be said about hot keys on other nodes, since
# they may be an arbitrary blend of old and new keys.
for i in $(seq 1 3) ; do
	run_fetch_ring 5 $((100 + i))

	check_hot_keys 0 101 "$i"

	echo
done