1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
|
#!@BASH_SHELL@
#
# Copyright (C) 1997-2003 Sistina Software, Inc. All rights reserved.
# Copyright (C) 2004-2011 Red Hat, Inc. All rights reserved.
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
# lv_verify
#
# Verify the parameters passed in
#
lv_verify()
{
# Anything to verify? Perhaps the names?
return $OCF_SUCCESS
}
# lv_owner
#
# Returns:
# 1 == We are the owner
# 2 == We can claim it
# 0 == Owned by someone else
function lv_owner
{
local my_name=$1
local owner=$2
if [ -z "$my_name" ]; then
ocf_log err "Unable to determine cluster node name"
return 0
fi
if [ -z "$owner" ]; then
# No-one owns this LV yet, so we can claim it
return 2
fi
if [ $owner != $my_name ]; then
if is_node_member_clustat $owner ; then
return 0
fi
return 2
fi
return 1
}
steal_tag()
{
local owner=$1
local lv_path=$2
ocf_log notice "Owner of $lv_path is not in the cluster"
ocf_log notice "Stealing $lv_path"
lvchange --deltag $owner $lv_path
if [ $? -ne 0 ]; then
ocf_log err "Failed to steal $lv_path from $owner"
return $OCF_ERR_GENERIC
fi
# Warning --deltag doesn't always result in failure
if [ ! -z `lvs -o tags --noheadings $lv_path` ]; then
ocf_log err "Failed to steal $lv_path from $owner."
return $OCF_ERR_GENERIC
fi
return $OCF_SUCCESS
}
restore_transient_failed_pvs()
{
local a=0
local -a results
results=(`pvs -o name,vg_name,attr --noheadings | grep $OCF_RESKEY_vg_name | grep -v 'unknown device'`)
while [ ! -z "${results[$a]}" ] ; do
if [[ ${results[$(($a + 2))]} =~ ..m ]] &&
[ $OCF_RESKEY_vg_name == ${results[$(($a + 1))]} ]; then
ocf_log notice "Attempting to restore missing PV, ${results[$a]} in $OCF_RESKEY_vg_name"
vgextend --restoremissing $OCF_RESKEY_vg_name ${results[$a]}
if [ $? -ne 0 ]; then
ocf_log notice "Failed to restore ${results[$a]}"
else
ocf_log notice " ${results[$a]} restored"
fi
fi
a=$(($a + 3))
done
}
# lv_exec_resilient
#
# Sometimes, devices can come back. Their metadata will conflict
# with the good devices that remain. This function filters out those
# failed devices when executing the given command
#
# Finishing with vgscan resets the cache/filter
lv_exec_resilient()
{
declare command=$1
declare all_pvs
ocf_log notice "Making resilient : $command"
if [ -z "$command" ]; then
ocf_log err "lv_exec_resilient: Arguments not supplied"
return $OCF_ERR_ARGS
fi
# pvs will print out only those devices that are valid
# If a device dies and comes back, it will not appear
# in pvs output (but you will get a Warning).
all_pvs=(`pvs --noheadings -o pv_name | grep -v Warning`)
# Now we use those valid devices in a filter which we set up.
# The device will then be activated because there are no
# metadata conflicts.
command=$command" --config devices{filter=["
for i in ${all_pvs[*]}; do
command=$command'"a|'$i'|",'
done
command=$command"\"r|.*|\"]}"
ocf_log notice "Resilient command: $command"
if ! $command ; then
ocf_log err "lv_exec_resilient failed"
vgscan
return $OCF_ERR_GENERIC
else
vgscan
return $OCF_SUCCESS
fi
}
# lv_activate_resilient
#
# Sometimes, devices can come back. Their metadata will conflict
# with the good devices that remain. We must filter out those
# failed devices when trying to reactivate
lv_activate_resilient()
{
declare action=$1
declare lv_path=$2
declare op="-ay"
if [ -z "$action" ] || [ -z "$lv_path" ]; then
ocf_log err "lv_activate_resilient: Arguments not supplied"
return $OCF_ERR_ARGS
fi
if [ $action != "start" ]; then
op="-an"
elif [[ "$(lvs -o attr --noheadings $lv_path)" =~ r.......p ]] ||
[[ "$(lvs -o attr --noheadings $lv_path)" =~ R.......p ]]; then
# We can activate partial RAID LVs and run just fine.
ocf_log notice "Attempting activation of partial RAID LV, $lv_path"
op="-ay --partial"
fi
if ! lv_exec_resilient "lvchange $op $lv_path" ; then
ocf_log err "lv_activate_resilient $action failed on $lv_path"
return $OCF_ERR_GENERIC
else
return $OCF_SUCCESS
fi
}
lv_status_clustered()
{
declare lv_path="$OCF_RESKEY_vg_name/$OCF_RESKEY_lv_name"
#
# Check if device is active
#
if [[ ! "$(lvs -o attr --noheadings $lv_path)" =~ ....a. ]]; then
return $OCF_NOT_RUNNING
fi
return $OCF_SUCCESS
}
# lv_status
#
# Is the LV active?
lv_status_single()
{
declare lv_path="$OCF_RESKEY_vg_name/$OCF_RESKEY_lv_name"
declare dev="/dev/$lv_path"
declare realdev
declare owner
declare my_name
#
# Check if device is active
#
if [[ ! "$(lvs -o attr --noheadings $lv_path)" =~ ....a. ]]; then
return $OCF_NOT_RUNNING
fi
if [[ "$(vgs -o attr --noheadings $OCF_RESKEY_vg_name)" =~ .....c ]]; then
ocf_log notice "$OCF_RESKEY_vg_name is a cluster volume. Ignoring..."
return $OCF_SUCCESS
fi
#
# Check if all links/device nodes are present
#
if [ -h "$dev" ]; then
realdev=$(readlink -f $dev)
if [ $? -ne 0 ]; then
ocf_log err "Failed to follow link, $dev"
return $OCF_ERR_ARGS
fi
if [ ! -b $realdev ]; then
ocf_log err "Device node for $lv_path is not present"
return $OCF_ERR_GENERIC
fi
else
ocf_log err "Symbolic link for $lv_path is not present"
return $OCF_ERR_GENERIC
fi
#
# Verify that we are the correct owner
#
owner=`lvs -o tags --noheadings $lv_path | tr -d ' '`
my_name=$(local_node_name)
if [ -z "$my_name" ]; then
ocf_log err "Unable to determine local machine name"
# FIXME: I don't really want to fail on 1st offense
return $OCF_SUCCESS
fi
if [ -z "$owner" ] || [ "$my_name" != "$owner" ]; then
ocf_log err "WARNING: $lv_path should not be active"
ocf_log err "WARNING: $my_name does not own $lv_path"
ocf_log err "WARNING: Attempting shutdown of $lv_path"
lv_activate_resilient "stop" $lv_path
return $OCF_ERR_GENERIC
fi
return $OCF_SUCCESS
}
function lv_status
{
# We pass in the VG name to see of the logical volume is clustered
if [[ $(vgs -o attr --noheadings $OCF_RESKEY_vg_name) =~ .....c ]]; then
lv_status_clustered
else
lv_status_single
fi
}
# lv_activate_and_tag
lv_activate_and_tag()
{
declare action=$1
declare tag=$2
declare lv_path=$3
typeset self_fence=""
case ${OCF_RESKEY_self_fence} in
"yes") self_fence=1 ;;
1) self_fence=1 ;;
*) self_fence="" ;;
esac
if [ -z "$action" ] || [ -z "$tag" ] || [ -z "$lv_path" ]; then
ocf_log err "Supplied args: 1) $action, 2) $tag, 3) $lv_path"
return $OCF_ERR_ARGS
fi
if [ "$action" == "start" ]; then
ocf_log notice "Activating $lv_path"
lvchange --addtag $tag $lv_path
if [ $? -ne 0 ]; then
ocf_log err "Unable to add tag to $lv_path"
return $OCF_ERR_GENERIC
fi
if ! lv_activate_resilient $action $lv_path; then
ocf_log err "Unable to activate $lv_path"
return $OCF_ERR_GENERIC
fi
else
ocf_log notice "Deactivating $lv_path"
if ! lv_activate_resilient $action $lv_path; then
if [ "$self_fence" ]; then
ocf_log err "Unable to deactivate $lv_path: REBOOTING"
sync
reboot -fn
else
ocf_log err "Unable to deactivate $lv_path"
fi
return $OCF_ERR_GENERIC
fi
# Only try to remove tag if it is our tag
if [ "`lvs --noheadings -o lv_tags $lv_path | tr -d ' '`" == $tag ]; then
ocf_log notice "Removing ownership tag ($tag) from $lv_path"
lvchange --deltag $tag $lv_path
if [ $? -ne 0 ]; then
ocf_log err "Unable to delete tag from $lv_path"
# Newer versions of LVM require the missing PVs to
# be removed from the VG via a separate call before
# the tag can be removed.
ocf_log err "Attempting volume group clean-up and retry"
vgreduce --removemissing --mirrorsonly --force $OCF_RESKEY_vg_name
# Retry tag deletion
lvchange --deltag $tag $lv_path
if [ $? -ne 0 ]; then
if [ "$self_fence" ]; then
ocf_log err "Failed to delete tag from $lv_path: REBOOTING"
sync
reboot -fn
else
ocf_log err "Failed to delete tag from $lv_path"
fi
return $OCF_ERR_GENERIC
fi
fi
fi
fi
return $OCF_SUCCESS
}
# lv_activate
# $1: start/stop only
#
# Basically, if we want to [de]activate an LVM volume,
# we must own it. That means that our tag must be on it.
# This requires a change to /etc/lvm/lvm.conf:
# volume_list = [ "root_volume", "@my_hostname" ]
# where "root_volume" is your root volume group and
# "my_hostname" is $(local_node_name)
#
# If there is a node failure, we may wish to "steal" the
# LV. For that, we need to check if the node that owns
# it is still part of the cluster. We use the tag to
# determine who owns the volume then query for their
# liveness. If they are dead, we can steal.
lv_activate()
{
declare lv_path="$OCF_RESKEY_vg_name/$OCF_RESKEY_lv_name"
declare owner=`lvs -o tags --noheadings $lv_path | tr -d ' '`
declare my_name=$(local_node_name)
local owned
lv_owner $my_name $owner
owned=$?
if [ $owned -eq 0 ]; then
ocf_log info "Someone else owns this logical volume"
return $OCF_ERR_GENERIC
fi
# If this is a partial VG, attempt to
# restore any transiently failed PVs
if [[ $(vgs -o attr --noheadings $OCF_RESKEY_vg_name) =~ ...p ]]; then
ocf_log err "Volume group \"$OCF_RESKEY_vg_name\" has PVs marked as missing"
restore_transient_failed_pvs
fi
if [ ! -z "$owner" ] && [ $owned -eq 2 ]; then
steal_tag $owner $lv_path
fi
if ! lv_activate_and_tag $1 $my_name $lv_path; then
ocf_log err "Failed to $1 $lv_path"
ocf_log notice "Attempting cleanup of $OCF_RESKEY_vg_name"
if vgreduce --removemissing --mirrorsonly --force --config \
"activation { volume_list = \"$OCF_RESKEY_vg_name\" }" \
$OCF_RESKEY_vg_name; then
ocf_log notice "$OCF_RESKEY_vg_name now consistent"
owner=`lvs -o tags --noheadings $lv_path | tr -d ' '`
lv_owner $my_name $owner
owned=$?
if [ ! -z "$owner" ] && [ $owned -eq 2 ]; then
steal_tag $owner $lv_path
ret=$?
if [ $ret -ne $OCF_SUCCESS ]; then
return $ret
fi
elif [ $owned -eq 0 ]; then
ocf_log info "Someone else owns this logical volume"
return $OCF_ERR_GENERIC
fi
if ! lv_activate_and_tag $1 $my_name $lv_path; then
ocf_log err "Failed second attempt to $1 $lv_path"
return $OCF_ERR_GENERIC
else
ocf_log notice "Second attempt to $1 $lv_path successful"
return $OCF_SUCCESS
fi
else
ocf_log err "Failed to $1 $lv_path"
return $OCF_ERR_GENERIC
fi
fi
return $OCF_SUCCESS
}
function lv_start_clustered
{
if lvchange -aey $OCF_RESKEY_vg_name/$OCF_RESKEY_lv_name; then
return $OCF_SUCCESS
fi
# FAILED exclusive activation:
# This can be caused by an LV being active remotely.
# Before attempting a repair effort, we should attempt
# to deactivate the LV cluster-wide; but only if the LV
# is not open. Otherwise, it is senseless to attempt.
if ! [[ "$(lvs -o attr --noheadings $OCF_RESKEY_vg_name/$OCF_RESKEY_lv_name)" =~ ....ao ]]; then
# We'll wait a small amount of time for some settling before
# attempting to deactivate. Then the deactivate will be
# immediately followed by another exclusive activation attempt.
sleep 5
if ! lvchange -an $OCF_RESKEY_vg_name/$OCF_RESKEY_lv_name; then
# Someone could have the device open.
# We can't do anything about that.
ocf_log err "Unable to perform required deactivation of $OCF_RESKEY_vg_name/$OCF_RESKEY_lv_name before starting"
return $OCF_ERR_GENERIC
fi
if lvchange -aey $OCF_RESKEY_vg_name/$OCF_RESKEY_lv_name; then
# Second attempt after deactivation was successful, we now
# have the lock exclusively
return $OCF_SUCCESS
fi
fi
# Failed to activate:
# This could be due to a device failure (or another machine could
# have snuck in between the deactivation/activation). We don't yet
# have a mechanism to check for remote activation, so we will proceed
# with repair action.
ocf_log err "Failed to activate logical volume, $OCF_RESKEY_vg_name/$OCF_RESKEY_lv_name"
ocf_log notice "Attempting cleanup of $OCF_RESKEY_vg_name/$OCF_RESKEY_lv_name"
if ! lvconvert --repair --use-policies $OCF_RESKEY_vg_name/$OCF_RESKEY_lv_name; then
ocf_log err "Failed to cleanup $OCF_RESKEY_vg_name/$OCF_RESKEY_lv_name"
return $OCF_ERR_GENERIC
fi
if ! lvchange -aey $OCF_RESKEY_vg_name/$OCF_RESKEY_lv_name; then
ocf_log err "Failed second attempt to activate $OCF_RESKEY_vg_name/$OCF_RESKEY_lv_name"
return $OCF_ERR_GENERIC
fi
ocf_log notice "Second attempt to activate $OCF_RESKEY_vg_name/$OCF_RESKEY_lv_name successful"
return $OCF_SUCCESS
}
function lv_start_single
{
if ! lvs $OCF_RESKEY_vg_name >& /dev/null; then
lv_count=0
else
lv_count=`lvs --noheadings -o name $OCF_RESKEY_vg_name | grep -v _mlog | grep -v _mimage | grep -v nconsistent | wc -l`
fi
if [ $lv_count -gt 1 ]; then
ocf_log err "HA LVM requires Only one logical volume per volume group."
ocf_log err "There are currently $lv_count logical volumes in $OCF_RESKEY_vg_name"
ocf_log err "Failing HA LVM start of $OCF_RESKEY_vg_name/$OCF_RESKEY_lv_name"
exit $OCF_ERR_GENERIC
fi
if ! lv_activate start; then
return 1
fi
return 0
}
function lv_start
{
# We pass in the VG name to see of the logical volume is clustered
if [[ "$(vgs -o attr --noheadings $OCF_RESKEY_vg_name)" =~ .....c ]]; then
lv_start_clustered
else
lv_start_single
fi
}
function lv_stop_clustered
{
lvchange -aln $OCF_RESKEY_vg_name/$OCF_RESKEY_lv_name
}
function lv_stop_single
{
if ! lv_activate stop; then
return 1
fi
return 0
}
function lv_stop
{
# We pass in the VG name to see of the logical volume is clustered
if [[ "$(vgs -o attr --noheadings $OCF_RESKEY_vg_name)" =~ .....c ]]; then
lv_stop_clustered
else
lv_stop_single
fi
}
|