blob: 280714e4e587c61cf80f15f33d832dcd3ec76d4e (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
|
tasks:
- cephadm.shell:
env: [sha1]
mon.a:
# setup rgw
- radosgw-admin realm create --rgw-realm=r --default
- radosgw-admin zonegroup create --rgw-zonegroup=default --master --default
- radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default
- radosgw-admin period update --rgw-realm=r --commit
- ceph orch apply rgw foo --realm r --zone z --placement=2 --port=8000
# setup iscsi
- ceph osd pool create foo
- rbd pool init foo
- ceph orch apply iscsi foo u p
- sleep 180
- ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force
- ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force
- ceph config set global log_to_journald false --force
# get some good info on the state of things pre-upgrade. Useful for debugging
- ceph orch ps
- ceph versions
- ceph -s
- ceph orch ls
# doing staggered upgrade requires mgr daemons being on a version that contains the staggered upgrade code
# until there is a stable version that contains it, we can test by manually upgrading a mgr daemon
- ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1
- ceph orch ps --refresh
- sleep 180
# gather more possible debugging info
- ceph orch ps
- ceph versions
- ceph -s
- ceph health detail
# check that there are two different versions found for mgr daemon (which implies we upgraded one)
- ceph versions | jq -e '.mgr | length == 2'
- ceph mgr fail
- sleep 180
# now try upgrading the other mgr
- ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1
- ceph orch ps --refresh
- sleep 180
# gather more possible debugging info
- ceph orch ps
- ceph versions
- ceph health detail
- ceph -s
- ceph mgr fail
- sleep 180
# gather more debugging info
- ceph orch ps
- ceph versions
- ceph -s
- ceph health detail
# now that both mgrs should have been redeployed with the new version, we should be back on only 1 version for the mgrs
- ceph versions | jq -e '.mgr | length == 1'
- ceph mgr fail
- sleep 180
# debugging info
- ceph orch ps
- ceph orch ls
- ceph versions
# to make sure mgr daemons upgrade is fully completed, including being deployed by a mgr on a new version
# also serves as an early failure if manually upgrading the mgrs failed as --daemon-types won't be recognized
- ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr
- while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
# verify only one version found for mgrs and that their version hash matches what we are upgrading to
- ceph versions | jq -e '.mgr | length == 1'
- ceph versions | jq -e '.mgr | keys' | grep $sha1
# verify overall we still see two versions, basically to make sure --daemon-types wasn't ignored and all daemons upgraded
- ceph versions | jq -e '.overall | length == 2'
# check that exactly two daemons have been upgraded to the new image (our 2 mgr daemons)
- ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 2'
- ceph orch upgrade status
- ceph health detail
# upgrade only the mons on one of the two hosts
- ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '{print $2}')
- while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
- ceph orch ps
# verify two different version seen for mons
- ceph versions | jq -e '.mon | length == 2'
- ceph orch upgrade status
- ceph health detail
# upgrade mons on the other hosts
- ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '{print $2}')
- while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
- ceph orch ps
# verify all mons now on same version and version hash matches what we are upgrading to
- ceph versions | jq -e '.mon | length == 1'
- ceph versions | jq -e '.mon | keys' | grep $sha1
# verify exactly 5 daemons are now upgraded (2 mgrs, 3 mons)
- ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 5'
- ceph orch upgrade status
- ceph health detail
# upgrade exactly 2 osd daemons
- ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2
- while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
- ceph orch ps
# verify two different versions now seen for osds
- ceph versions | jq -e '.osd | length == 2'
# verify exactly 7 daemons have been upgraded (2 mgrs, 3 mons, 2 osds)
- ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 7'
- ceph orch upgrade status
- ceph health detail
# upgrade one more osd
- ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1
- while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
- ceph orch ps
- ceph versions | jq -e '.osd | length == 2'
# verify now 8 daemons have been upgraded
- ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 8'
# upgrade the rest of the osds
- ceph orch upgrade status
- ceph health detail
- ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd
- while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
- ceph orch ps
# verify all osds are now on same version and version hash matches what we are upgrading to
- ceph versions | jq -e '.osd | length == 1'
- ceph versions | jq -e '.osd | keys' | grep $sha1
- ceph orch upgrade status
- ceph health detail
# upgrade the rgw daemons using --services
- ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.foo
- while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
- ceph orch ps
# verify all rgw daemons on same version and version hash matches what we are upgrading to
- ceph versions | jq -e '.rgw | length == 1'
- ceph versions | jq -e '.rgw | keys' | grep $sha1
- ceph orch upgrade status
- ceph health detail
# run upgrade one more time with no filter parameters to make sure anything left gets upgraded
- ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1
|
