blob: 5441b79f3885817101c6afc12ff4fe8c096223f0 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
|
# Check the manual failover
source "../tests/includes/init-tests.tcl"
test "Create a 5 nodes cluster" {
create_cluster 5 5
}
test "Cluster is up" {
assert_cluster_state ok
}
test "Cluster is writable" {
cluster_write_test 0
}
test "Instance #5 is a slave" {
assert {[RI 5 role] eq {slave}}
}
test "Instance #5 synced with the master" {
wait_for_condition 1000 50 {
[RI 5 master_link_status] eq {up}
} else {
fail "Instance #5 master link status is not up"
}
}
set current_epoch [CI 1 cluster_current_epoch]
set numkeys 50000
set numops 10000
set cluster [redis_cluster 127.0.0.1:[get_instance_attrib redis 0 port]]
catch {unset content}
array set content {}
test "Send CLUSTER FAILOVER to #5, during load" {
for {set j 0} {$j < $numops} {incr j} {
# Write random data to random list.
set listid [randomInt $numkeys]
set key "key:$listid"
set ele [randomValue]
# We write both with Lua scripts and with plain commands.
# This way we are able to stress Lua -> Redis command invocation
# as well, that has tests to prevent Lua to write into wrong
# hash slots.
if {$listid % 2} {
$cluster rpush $key $ele
} else {
$cluster eval {redis.call("rpush",KEYS[1],ARGV[1])} 1 $key $ele
}
lappend content($key) $ele
if {($j % 1000) == 0} {
puts -nonewline W; flush stdout
}
if {$j == $numops/2} {R 5 cluster failover}
}
}
test "Wait for failover" {
wait_for_condition 1000 50 {
[CI 1 cluster_current_epoch] > $current_epoch
} else {
fail "No failover detected"
}
}
test "Cluster should eventually be up again" {
assert_cluster_state ok
}
test "Cluster is writable" {
cluster_write_test 1
}
test "Instance #5 is now a master" {
assert {[RI 5 role] eq {master}}
}
test "Verify $numkeys keys for consistency with logical content" {
# Check that the Redis Cluster content matches our logical content.
foreach {key value} [array get content] {
assert {[$cluster lrange $key 0 -1] eq $value}
}
}
test "Instance #0 gets converted into a slave" {
wait_for_condition 1000 50 {
[RI 0 role] eq {slave}
} else {
fail "Old master was not converted into slave"
}
}
## Check that manual failover does not happen if we can't talk with the master.
source "../tests/includes/init-tests.tcl"
test "Create a 5 nodes cluster" {
create_cluster 5 5
}
test "Cluster is up" {
assert_cluster_state ok
}
test "Cluster is writable" {
cluster_write_test 0
}
test "Instance #5 is a slave" {
assert {[RI 5 role] eq {slave}}
}
test "Instance #5 synced with the master" {
wait_for_condition 1000 50 {
[RI 5 master_link_status] eq {up}
} else {
fail "Instance #5 master link status is not up"
}
}
test "Make instance #0 unreachable without killing it" {
R 0 deferred 1
R 0 DEBUG SLEEP 10
}
test "Send CLUSTER FAILOVER to instance #5" {
R 5 cluster failover
}
test "Instance #5 is still a slave after some time (no failover)" {
after 5000
assert {[RI 5 role] eq {master}}
}
test "Wait for instance #0 to return back alive" {
R 0 deferred 0
assert {[R 0 read] eq {OK}}
}
## Check with "force" failover happens anyway.
source "../tests/includes/init-tests.tcl"
test "Create a 5 nodes cluster" {
create_cluster 5 5
}
test "Cluster is up" {
assert_cluster_state ok
}
test "Cluster is writable" {
cluster_write_test 0
}
test "Instance #5 is a slave" {
assert {[RI 5 role] eq {slave}}
}
test "Instance #5 synced with the master" {
wait_for_condition 1000 50 {
[RI 5 master_link_status] eq {up}
} else {
fail "Instance #5 master link status is not up"
}
}
test "Make instance #0 unreachable without killing it" {
R 0 deferred 1
R 0 DEBUG SLEEP 10
}
test "Send CLUSTER FAILOVER to instance #5" {
R 5 cluster failover force
}
test "Instance #5 is a master after some time" {
wait_for_condition 1000 50 {
[RI 5 role] eq {master}
} else {
fail "Instance #5 is not a master after some time regardless of FORCE"
}
}
test "Wait for instance #0 to return back alive" {
R 0 deferred 0
assert {[R 0 read] eq {OK}}
}
|
