blob: 18a26bda63b1720f76f224bbe0df3bb415c3e6e0 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
|
# Failover stress test.
# In this test a different node is killed in a loop for N
# iterations. The test checks that certain properties
# are preserved across iterations.
source "../tests/includes/init-tests.tcl"
source "../../../tests/support/cli.tcl"
test "Create a 5 nodes cluster" {
create_cluster 5 5
}
test "Cluster is up" {
assert_cluster_state ok
}
test "Enable AOF in all the instances" {
foreach_redis_id id {
R $id config set appendonly yes
# We use "appendfsync no" because it's fast but also guarantees that
# write(2) is performed before replying to client.
R $id config set appendfsync no
}
foreach_redis_id id {
wait_for_condition 1000 500 {
[RI $id aof_rewrite_in_progress] == 0 &&
[RI $id aof_enabled] == 1
} else {
fail "Failed to enable AOF on instance #$id"
}
}
}
# Return non-zero if the specified PID is about a process still in execution,
# otherwise 0 is returned.
proc process_is_running {pid} {
# PS should return with an error if PID is non existing,
# and catch will return non-zero. We want to return non-zero if
# the PID exists, so we invert the return value with expr not operator.
expr {![catch {exec ps -p $pid}]}
}
# Our resharding test performs the following actions:
#
# - N commands are sent to the cluster in the course of the test.
# - Every command selects a random key from key:0 to key:MAX-1.
# - The operation RPUSH key <randomvalue> is performed.
# - Tcl remembers into an array all the values pushed to each list.
# - After N/2 commands, the resharding process is started in background.
# - The test continues while the resharding is in progress.
# - At the end of the test, we wait for the resharding process to stop.
# - Finally the keys are checked to see if they contain the value they should.
set numkeys 50000
set numops 200000
set start_node_port [get_instance_attrib redis 0 port]
set cluster [redis_cluster 127.0.0.1:$start_node_port]
if {$::tls} {
# setup a non-TLS cluster client to the TLS cluster
set plaintext_port [get_instance_attrib redis 0 plaintext-port]
set cluster_plaintext [redis_cluster 127.0.0.1:$plaintext_port 0]
puts "Testing TLS cluster on start node 127.0.0.1:$start_node_port, plaintext port $plaintext_port"
} else {
set cluster_plaintext $cluster
puts "Testing using non-TLS cluster"
}
catch {unset content}
array set content {}
set tribpid {}
test "Cluster consistency during live resharding" {
set ele 0
for {set j 0} {$j < $numops} {incr j} {
# Trigger the resharding once we execute half the ops.
if {$tribpid ne {} &&
($j % 10000) == 0 &&
![process_is_running $tribpid]} {
set tribpid {}
}
if {$j >= $numops/2 && $tribpid eq {}} {
puts -nonewline "...Starting resharding..."
flush stdout
set target [dict get [get_myself [randomInt 5]] id]
set tribpid [lindex [exec \
../../../src/redis-cli --cluster reshard \
127.0.0.1:[get_instance_attrib redis 0 port] \
--cluster-from all \
--cluster-to $target \
--cluster-slots 100 \
--cluster-yes \
{*}[rediscli_tls_config "../../../tests"] \
| [info nameofexecutable] \
../tests/helpers/onlydots.tcl \
&] 0]
}
# Write random data to random list.
set listid [randomInt $numkeys]
set key "key:$listid"
incr ele
# We write both with Lua scripts and with plain commands.
# This way we are able to stress Lua -> Redis command invocation
# as well, that has tests to prevent Lua to write into wrong
# hash slots.
# We also use both TLS and plaintext connections.
if {$listid % 3 == 0} {
$cluster rpush $key $ele
} elseif {$listid % 3 == 1} {
$cluster_plaintext rpush $key $ele
} else {
$cluster eval {redis.call("rpush",KEYS[1],ARGV[1])} 1 $key $ele
}
lappend content($key) $ele
if {($j % 1000) == 0} {
puts -nonewline W; flush stdout
}
}
# Wait for the resharding process to end
wait_for_condition 1000 500 {
[process_is_running $tribpid] == 0
} else {
fail "Resharding is not terminating after some time."
}
}
test "Verify $numkeys keys for consistency with logical content" {
# Check that the Redis Cluster content matches our logical content.
foreach {key value} [array get content] {
if {[$cluster lrange $key 0 -1] ne $value} {
fail "Key $key expected to hold '$value' but actual content is [$cluster lrange $key 0 -1]"
}
}
}
test "Terminate and restart all the instances" {
foreach_redis_id id {
# Stop AOF so that an initial AOFRW won't prevent the instance from terminating
R $id config set appendonly no
kill_instance redis $id
restart_instance redis $id
}
}
test "Cluster should eventually be up again" {
assert_cluster_state ok
}
test "Verify $numkeys keys after the restart" {
# Check that the Redis Cluster content matches our logical content.
foreach {key value} [array get content] {
if {[$cluster lrange $key 0 -1] ne $value} {
fail "Key $key expected to hold '$value' but actual content is [$cluster lrange $key 0 -1]"
}
}
}
test "Disable AOF in all the instances" {
foreach_redis_id id {
R $id config set appendonly no
}
}
test "Verify slaves consistency" {
set verified_masters 0
foreach_redis_id id {
set role [R $id role]
lassign $role myrole myoffset slaves
if {$myrole eq {slave}} continue
set masterport [get_instance_attrib redis $id port]
set masterdigest [R $id debug digest]
foreach_redis_id sid {
set srole [R $sid role]
if {[lindex $srole 0] eq {master}} continue
if {[lindex $srole 2] != $masterport} continue
wait_for_condition 1000 500 {
[R $sid debug digest] eq $masterdigest
} else {
fail "Master and slave data digest are different"
}
incr verified_masters
}
}
assert {$verified_masters >= 5}
}
test "Dump sanitization was skipped for migrations" {
set verified_masters 0
foreach_redis_id id {
assert {[RI $id dump_payload_sanitizations] == 0}
}
}
|
