blob: 46c22a91347f44fa311f3d6c854eb8bc6fa8dcf0 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
|
# Failover stress test.
# In this test a different node is killed in a loop for N
# iterations. The test checks that certain properties
# are preserved across iterations.
source "../tests/includes/init-tests.tcl"
test "Create a 5 nodes cluster" {
create_cluster 5 5
}
test "Cluster is up" {
assert_cluster_state ok
}
set iterations 20
set cluster [redis_cluster 127.0.0.1:[get_instance_attrib redis 0 port]]
while {[incr iterations -1]} {
set tokill [randomInt 10]
set other [expr {($tokill+1)%10}] ; # Some other instance.
set key [randstring 20 20 alpha]
set val [randstring 20 20 alpha]
set role [RI $tokill role]
if {$role eq {master}} {
set slave {}
set myid [dict get [get_myself $tokill] id]
foreach_redis_id id {
if {$id == $tokill} continue
if {[dict get [get_myself $id] slaveof] eq $myid} {
set slave $id
}
}
if {$slave eq {}} {
fail "Unable to retrieve slave's ID for master #$tokill"
}
}
puts "--- Iteration $iterations ---"
if {$role eq {master}} {
test "Wait for slave of #$tokill to sync" {
wait_for_condition 1000 50 {
[string match {*state=online*} [RI $tokill slave0]]
} else {
fail "Slave of node #$tokill is not ok"
}
}
set slave_config_epoch [CI $slave cluster_my_epoch]
}
test "Cluster is writable before failover" {
for {set i 0} {$i < 100} {incr i} {
catch {$cluster set $key:$i $val:$i} err
assert {$err eq {OK}}
}
# Wait for the write to propagate to the slave if we
# are going to kill a master.
if {$role eq {master}} {
R $tokill wait 1 20000
}
}
test "Terminating node #$tokill" {
# Stop AOF so that an initial AOFRW won't prevent the instance from terminating
R $tokill config set appendonly no
kill_instance redis $tokill
}
if {$role eq {master}} {
test "Wait failover by #$slave with old epoch $slave_config_epoch" {
wait_for_condition 1000 50 {
[CI $slave cluster_my_epoch] > $slave_config_epoch
} else {
fail "No failover detected, epoch is still [CI $slave cluster_my_epoch]"
}
}
}
test "Cluster should eventually be up again" {
assert_cluster_state ok
}
test "Cluster is writable again" {
for {set i 0} {$i < 100} {incr i} {
catch {$cluster set $key:$i:2 $val:$i:2} err
assert {$err eq {OK}}
}
}
test "Restarting node #$tokill" {
restart_instance redis $tokill
}
test "Instance #$tokill is now a slave" {
wait_for_condition 1000 50 {
[RI $tokill role] eq {slave}
} else {
fail "Restarted instance is not a slave"
}
}
test "We can read back the value we set before" {
for {set i 0} {$i < 100} {incr i} {
catch {$cluster get $key:$i} err
assert {$err eq "$val:$i"}
catch {$cluster get $key:$i:2} err
assert {$err eq "$val:$i:2"}
}
}
}
test "Post condition: current_epoch >= my_epoch everywhere" {
foreach_redis_id id {
assert {[CI $id cluster_current_epoch] >= [CI $id cluster_my_epoch]}
}
}
|
