1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
|
proc get_slot_field {slot_output shard_id node_id attrib_id} {
return [lindex [lindex [lindex $slot_output $shard_id] $node_id] $attrib_id]
}
# Start a cluster with 3 masters and 4 replicas.
# These tests rely on specific node ordering, so make sure no node fails over.
start_cluster 3 4 {tags {external:skip cluster} overrides {cluster-replica-no-failover yes}} {
test "Set cluster hostnames and verify they are propagated" {
for {set j 0} {$j < [llength $::servers]} {incr j} {
R $j config set cluster-announce-hostname "host-$j.com"
}
wait_for_condition 50 100 {
[are_hostnames_propagated "host-*.com"] eq 1
} else {
fail "cluster hostnames were not propagated"
}
# Now that everything is propagated, assert everyone agrees
wait_for_cluster_propagation
}
test "Update hostnames and make sure they are all eventually propagated" {
for {set j 0} {$j < [llength $::servers]} {incr j} {
R $j config set cluster-announce-hostname "host-updated-$j.com"
}
wait_for_condition 50 100 {
[are_hostnames_propagated "host-updated-*.com"] eq 1
} else {
fail "cluster hostnames were not propagated"
}
# Now that everything is propagated, assert everyone agrees
wait_for_cluster_propagation
}
test "Remove hostnames and make sure they are all eventually propagated" {
for {set j 0} {$j < [llength $::servers]} {incr j} {
R $j config set cluster-announce-hostname ""
}
wait_for_condition 50 100 {
[are_hostnames_propagated ""] eq 1
} else {
fail "cluster hostnames were not propagated"
}
# Now that everything is propagated, assert everyone agrees
wait_for_cluster_propagation
}
test "Verify cluster-preferred-endpoint-type behavior for redirects and info" {
R 0 config set cluster-announce-hostname "me.com"
R 1 config set cluster-announce-hostname ""
R 2 config set cluster-announce-hostname "them.com"
wait_for_cluster_propagation
# Verify default behavior
set slot_result [R 0 cluster slots]
assert_equal "" [lindex [get_slot_field $slot_result 0 2 0] 1]
assert_equal "" [lindex [get_slot_field $slot_result 2 2 0] 1]
assert_equal "hostname" [lindex [get_slot_field $slot_result 0 2 3] 0]
assert_equal "me.com" [lindex [get_slot_field $slot_result 0 2 3] 1]
assert_equal "hostname" [lindex [get_slot_field $slot_result 2 2 3] 0]
assert_equal "them.com" [lindex [get_slot_field $slot_result 2 2 3] 1]
# Redirect will use the IP address
catch {R 0 set foo foo} redir_err
assert_match "MOVED * 127.0.0.1:*" $redir_err
# Verify prefer hostname behavior
R 0 config set cluster-preferred-endpoint-type hostname
set slot_result [R 0 cluster slots]
assert_equal "me.com" [get_slot_field $slot_result 0 2 0]
assert_equal "them.com" [get_slot_field $slot_result 2 2 0]
# Redirect should use hostname
catch {R 0 set foo foo} redir_err
assert_match "MOVED * them.com:*" $redir_err
# Redirect to an unknown hostname returns ?
catch {R 0 set barfoo bar} redir_err
assert_match "MOVED * ?:*" $redir_err
# Verify unknown hostname behavior
R 0 config set cluster-preferred-endpoint-type unknown-endpoint
# Verify default behavior
set slot_result [R 0 cluster slots]
assert_equal "ip" [lindex [get_slot_field $slot_result 0 2 3] 0]
assert_equal "127.0.0.1" [lindex [get_slot_field $slot_result 0 2 3] 1]
assert_equal "ip" [lindex [get_slot_field $slot_result 2 2 3] 0]
assert_equal "127.0.0.1" [lindex [get_slot_field $slot_result 2 2 3] 1]
assert_equal "ip" [lindex [get_slot_field $slot_result 1 2 3] 0]
assert_equal "127.0.0.1" [lindex [get_slot_field $slot_result 1 2 3] 1]
# Not required by the protocol, but IP comes before hostname
assert_equal "hostname" [lindex [get_slot_field $slot_result 0 2 3] 2]
assert_equal "me.com" [lindex [get_slot_field $slot_result 0 2 3] 3]
assert_equal "hostname" [lindex [get_slot_field $slot_result 2 2 3] 2]
assert_equal "them.com" [lindex [get_slot_field $slot_result 2 2 3] 3]
# This node doesn't have a hostname
assert_equal 2 [llength [get_slot_field $slot_result 1 2 3]]
# Redirect should use empty string
catch {R 0 set foo foo} redir_err
assert_match "MOVED * :*" $redir_err
R 0 config set cluster-preferred-endpoint-type ip
}
test "Verify the nodes configured with prefer hostname only show hostname for new nodes" {
# Have everyone forget node 6 and isolate it from the cluster.
isolate_node 6
# Set hostnames for the masters, now that the node is isolated
R 0 config set cluster-announce-hostname "shard-1.com"
R 1 config set cluster-announce-hostname "shard-2.com"
R 2 config set cluster-announce-hostname "shard-3.com"
# Prevent Node 0 and Node 6 from properly meeting,
# they'll hang in the handshake phase. This allows us to
# test the case where we "know" about it but haven't
# successfully retrieved information about it yet.
R 0 DEBUG DROP-CLUSTER-PACKET-FILTER 0
R 6 DEBUG DROP-CLUSTER-PACKET-FILTER 0
# Have a replica meet the isolated node
R 3 cluster meet 127.0.0.1 [srv -6 port]
# Wait for the isolated node to learn about the rest of the cluster,
# which correspond to a single entry in cluster nodes. Note this
# doesn't mean the isolated node has successfully contacted each
# node.
wait_for_condition 50 100 {
[llength [split [R 6 CLUSTER NODES] "\n"]] eq [expr [llength $::servers] + 1]
} else {
fail "Isolated node didn't learn about the rest of the cluster *"
}
# Now, we wait until the two nodes that aren't filtering packets
# to accept our isolated nodes connections. At this point they will
# start showing up in cluster slots.
wait_for_condition 50 100 {
[llength [R 6 CLUSTER SLOTS]] eq 2
} else {
fail "Node did not learn about the 2 shards it can talk to"
}
set slot_result [R 6 CLUSTER SLOTS]
assert_equal [lindex [get_slot_field $slot_result 0 2 3] 1] "shard-2.com"
assert_equal [lindex [get_slot_field $slot_result 1 2 3] 1] "shard-3.com"
# Also make sure we know about the isolated master, we
# just can't reach it.
set master_id [R 0 CLUSTER MYID]
assert_match "*$master_id*" [R 6 CLUSTER NODES]
# Stop dropping cluster packets, and make sure everything
# stabilizes
R 0 DEBUG DROP-CLUSTER-PACKET-FILTER -1
R 6 DEBUG DROP-CLUSTER-PACKET-FILTER -1
# This operation sometimes spikes to around 5 seconds to resolve the state,
# so it has a higher timeout.
wait_for_condition 50 500 {
[llength [R 6 CLUSTER SLOTS]] eq 3
} else {
fail "Node did not learn about the 2 shards it can talk to"
}
set slot_result [R 6 CLUSTER SLOTS]
assert_equal [lindex [get_slot_field $slot_result 0 2 3] 1] "shard-1.com"
assert_equal [lindex [get_slot_field $slot_result 1 2 3] 1] "shard-2.com"
assert_equal [lindex [get_slot_field $slot_result 2 2 3] 1] "shard-3.com"
}
test "Test restart will keep hostname information" {
# Set a new hostname, reboot and make sure it sticks
R 0 config set cluster-announce-hostname "restart-1.com"
# Store the hostname in the config
R 0 config rewrite
restart_server 0 true false
set slot_result [R 0 CLUSTER SLOTS]
assert_equal [lindex [get_slot_field $slot_result 0 2 3] 1] "restart-1.com"
# As a sanity check, make sure everyone eventually agrees
wait_for_cluster_propagation
}
test "Test hostname validation" {
catch {R 0 config set cluster-announce-hostname [string repeat x 256]} err
assert_match "*Hostnames must be less than 256 characters*" $err
catch {R 0 config set cluster-announce-hostname "?.com"} err
assert_match "*Hostnames may only contain alphanumeric characters, hyphens or dots*" $err
# Note this isn't a valid hostname, but it passes our internal validation
R 0 config set cluster-announce-hostname "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-."
}
}
|
