summaryrefslogtreecommitdiffstats
path: root/src/test/test_lost.sh
blob: d8246b27327b6f3243ac7976dc5d3a9cae5f2f7d (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
#!/usr/bin/env bash
set -x

#
# Test the lost object logic
#

# Includes
source "`dirname $0`/test_common.sh"

TEST_POOL=rbd

# Functions
setup() {
        export CEPH_NUM_OSD=$1
        vstart_config=$2

        # Start ceph
        ./stop.sh

        # set recovery start to a really long time to ensure that we don't start recovery
        ./vstart.sh -d -n -o "$vstart_config" || die "vstart failed"

	# for exiting pools set size not greater than number of OSDs,
	# so recovery from degraded ps is possible
	local changed=0
	for pool in `./ceph osd pool ls`; do
	    local size=`./ceph osd pool get ${pool} size | awk '{print $2}'`
	    if [ "${size}" -gt "${CEPH_NUM_OSD}" ]; then
		./ceph osd pool set ${pool} size ${CEPH_NUM_OSD}
		changed=1
	    fi
	done
	if [ ${changed} -eq 1 ]; then
	    # XXX: When a pool has degraded pgs due to size greater than number
	    # of OSDs, after decreasing the size the recovery still could stuck
	    # and requires an additional kick.
	    ./ceph osd out 0
	    ./ceph osd in 0
	fi

	poll_cmd "./ceph health" HEALTH_OK 1 30
}

recovery1_impl() {
        # Write lots and lots of objects
        write_objects 1 1 200 4000 $TEST_POOL

        # Take down osd1
        stop_osd 1

        # Continue writing a lot of objects
        write_objects 2 2 200 4000 $TEST_POOL

        # Bring up osd1
        restart_osd 1

        # Finish peering.
        sleep 15

        # Stop osd0.
        # At this point we have peered, but *NOT* recovered.
        # Objects should be lost.
        stop_osd 0

	poll_cmd "./ceph pg debug degraded_pgs_exist" TRUE 3 120
        [ $? -eq 1 ] || die "Failed to see degraded PGs."
	poll_cmd "./ceph pg debug unfound_objects_exist" TRUE 3 120
        [ $? -eq 1 ] || die "Failed to see unfound objects."
        echo "Got unfound objects."

        restart_osd 0
	sleep 20
	start_recovery 2

        # Turn on recovery and wait for it to complete.
	poll_cmd "./ceph pg debug unfound_objects_exist" FALSE 3 120
        [ $? -eq 1 ] || die "Failed to recover unfound objects."
	poll_cmd "./ceph pg debug degraded_pgs_exist" FALSE 3 120
        [ $? -eq 1 ] || die "Recovery never finished."
}

recovery1() {
        setup 2 'osd recovery delay start = 10000'
        recovery1_impl
}

lost1_impl() {
	local flags="$@"
	local lost_action=delete
	local pgs_unfound pg

	if is_set revert_lost $flags; then
	    lost_action=revert
	fi

        # Write lots and lots of objects
        write_objects 1 1 20 8000 $TEST_POOL

        # Take down osd1
        stop_osd 1

        # Continue writing a lot of objects
        write_objects 2 2 20 8000 $TEST_POOL

        # Bring up osd1
        restart_osd 1

        # Finish peering.
        sleep 15

        # Stop osd0.
        # At this point we have peered, but *NOT* recovered.
        # Objects should be lost.
        stop_osd 0

	# Since recovery can't proceed, stuff should be unfound.
	poll_cmd "./ceph pg debug unfound_objects_exist" TRUE 3 120
        [ $? -eq 1 ] || die "Failed to see unfound objects."

	pgs_unfound=`./ceph health detail |awk '$1 = "pg" && /[0-9] unfound$/ {print $2}'`

	[ -n "$pgs_unfound" ] || die "no pg with unfound objects"

	for pg in $pgs_unfound; do
	    ./ceph pg $pg mark_unfound_lost revert &&
	    die "mark_unfound_lost unexpectedly succeeded for pg $pg"
	done

	if ! is_set mark_osd_lost $flags && ! is_set rm_osd $flags; then
	    return
	fi

	if is_set try_to_fetch_unfound $flags; then
	  # Ask for an object while it's still unfound, and
	  # verify we get woken to an error when it's declared lost.
	  echo "trying to get one of the unfound objects"
	  (
	  ./rados -c ./ceph.conf -p $TEST_POOL get obj02 $TEMPDIR/obj02 &&\
	    die "expected radostool error"
	  ) &
	fi

	if is_set mark_osd_lost $flags; then
	  ./ceph osd lost 0 --yes-i-really-mean-it
	fi

	if is_set rm_osd $flags; then
	    ./ceph osd rm 0
	fi

	if ! is_set auto_mark_unfound_lost $flags; then
	    for pg in $pgs_unfound; do
		./ceph pg $pg mark_unfound_lost ${lost_action} ||
		  die "mark_unfound_lost failed for pg $pg"
	    done
	fi

	start_recovery 2

	# Unfound objects go away and are turned into lost objects.
	poll_cmd "./ceph pg debug unfound_objects_exist" FALSE 3 120
        [ $? -eq 1 ] || die "Unfound objects didn't go away."

	for pg in `ceph pg ls | awk '/^[0-9]/ {print $1}'`; do
	    ./ceph pg $pg mark_unfound_lost revert 2>&1 |
	      grep 'pg has no unfound objects' ||
	      die "pg $pg has unfound objects"
	done

	# Reading from a lost object gives back an error code.
	# TODO: check error code
	./rados -c ./ceph.conf -p $TEST_POOL get obj01 $TEMPDIR/obj01
	if [ lost_action = delete -a $? -eq 0 ]; then
	  die "expected radostool error"
	elif [ lost_action = revert -a $? -ne 0 ]; then
	  die "unexpected radostool error"
	fi

	if is_set try_to_fetch_unfound $flags; then
	  echo "waiting for the try_to_fetch_unfound \
radostool instance to finish"
	  wait
	fi
}

lost1() {
        setup 2 'osd recovery delay start = 10000'
        lost1_impl mark_osd_lost revert_lost
}

lost2() {
        setup 2 'osd recovery delay start = 10000'
        lost1_impl mark_osd_lost try_to_fetch_unfound
}

lost3() {
        setup 2 'osd recovery delay start = 10000'
        lost1_impl rm_osd
}

lost4() {
        setup 2 'osd recovery delay start = 10000'
        lost1_impl mark_osd_lost rm_osd
}

lost5() {
        setup 2 'osd recovery delay start = 10000'
        lost1_impl mark_osd_lost auto_mark_unfound_lost
}

all_osds_die_impl() {
        poll_cmd "./ceph osd stat" '3 up, 3 in' 20 240
        [ $? -eq 1 ] || die "didn't start 3 osds"

        stop_osd 0
        stop_osd 1
        stop_osd 2

	# wait for the MOSDPGStat timeout
        poll_cmd "./ceph osd stat" '0 up' 20 240
        [ $? -eq 1 ] || die "all osds weren't marked as down"
}

all_osds_die() {
	setup 3 'osd mon report interval = 3
	mon osd report timeout = 60'

	all_osds_die_impl
}

run() {
        recovery1 || die "test failed"

        lost1 || die "test failed"

	# XXX: try_to_fetch_unfound test currently hangs on "waiting for the
	# try_to_fetch_unfound radostool instance to finish"
	#lost2 || die "test failed"

	lost3 || die "test failed"

	lost4 || die "test failed"

	# XXX: automatically marking lost is not implemented
	#lost5 || die "test failed"

        all_osds_die || die "test failed"
}

if [ -z "$@" ]; then
	run
	echo OK
	exit 0
fi

$@