summaryrefslogtreecommitdiffstats
path: root/qa/standalone/mon/health-mute.sh
blob: d8e07ca061bb26f8e0115deece1d46fa62e0e725 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
#!/bin/bash

source $CEPH_ROOT/qa/standalone/ceph-helpers.sh

function run() {
    local dir=$1
    shift

    export CEPH_MON="127.0.0.1:7143" # git grep '\<714\>' : there must be only one
    export CEPH_ARGS
    CEPH_ARGS+="--fsid=$(uuidgen) --auth-supported=none --mon-pg-warn-min-per-osd 0 --mon-max-pg-per-osd 1000 "
    CEPH_ARGS+="--mon-host=$CEPH_MON "

    local funcs=${@:-$(set | sed -n -e 's/^\(TEST_[0-9a-z_]*\) .*/\1/p')}
    for func in $funcs ; do
        setup $dir || return 1
        $func $dir || return 1
        teardown $dir || return 1
    done
}

function TEST_mute() {
    local dir=$1
    setup $dir || return 1

    set -o pipefail

    run_mon $dir a || return 1
    run_mgr $dir x || return 1
    run_osd $dir 0 || return 1
    run_osd $dir 1 || return 1
    run_osd $dir 2 || return 1

    ceph osd pool create foo 8
    ceph osd pool application enable foo rbd --yes-i-really-mean-it
    wait_for_clean || return 1

    ceph -s
    ceph health | grep HEALTH_OK || return 1
    # test warning on setting pool size=1
    ceph osd pool set foo size 1 --yes-i-really-mean-it
    ceph -s
    ceph health | grep HEALTH_WARN || return 1
    ceph health detail | grep POOL_NO_REDUNDANCY || return 1
    ceph health mute POOL_NO_REDUNDANCY
    ceph -s
    ceph health | grep HEALTH_OK | grep POOL_NO_REDUNDANCY || return 1
    ceph health unmute POOL_NO_REDUNDANCY
    ceph -s
    ceph health | grep HEALTH_WARN || return 1
    # restore pool size to default
    ceph osd pool set foo size 3
    ceph -s
    ceph health | grep HEALTH_OK || return 1
    ceph osd set noup
    ceph -s
    ceph health detail | grep OSDMAP_FLAGS || return 1
    ceph osd down 0
    ceph -s
    ceph health detail | grep OSD_DOWN || return 1
    ceph health detail | grep HEALTH_WARN || return 1

    ceph health mute OSD_DOWN
    ceph health mute OSDMAP_FLAGS
    ceph -s
    ceph health | grep HEALTH_OK | grep OSD_DOWN | grep OSDMAP_FLAGS || return 1
    ceph health unmute OSD_DOWN
    ceph -s
    ceph health | grep HEALTH_WARN || return 1

    # ttl
    ceph health mute OSD_DOWN 10s
    ceph -s
    ceph health | grep HEALTH_OK || return 1
    sleep 15
    ceph -s
    ceph health | grep HEALTH_WARN || return 1

    # sticky
    ceph health mute OSDMAP_FLAGS --sticky
    ceph osd unset noup
    sleep 5
    ceph -s
    ceph health | grep OSDMAP_FLAGS || return 1
    ceph osd set noup
    ceph -s
    ceph health | grep HEALTH_OK || return 1

    # rachet down on OSD_DOWN count
    ceph osd down 0 1
    ceph -s
    ceph health detail | grep OSD_DOWN || return 1

    ceph health mute OSD_DOWN
    kill_daemons $dir TERM osd.0
    ceph osd unset noup
    sleep 10
    ceph -s
    ceph health detail | grep OSD_DOWN || return 1
    ceph health detail | grep '1 osds down' || return 1
    ceph health | grep HEALTH_OK || return 1

    sleep 10 # give time for mon tick to rachet the mute
    ceph osd set noup
    ceph health mute OSDMAP_FLAGS
    ceph -s
    ceph health detail
    ceph health | grep HEALTH_OK || return 1

    ceph osd down 1
    ceph -s
    ceph health detail
    ceph health detail | grep '2 osds down' || return 1

    sleep 10 # give time for mute to clear
    ceph -s
    ceph health detail
    ceph health | grep HEALTH_WARN || return 1
    ceph health detail | grep '2 osds down' || return 1

    teardown $dir || return 1
}

main health-mute "$@"