1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
|
import logging
import signal
import time
from gevent import sleep
from gevent.greenlet import Greenlet
from gevent.event import Event
log = logging.getLogger(__name__)
class DaemonWatchdog(Greenlet):
"""
DaemonWatchdog::
Watch Ceph daemons for failures. If an extended failure is detected (i.e.
not intentional), then the watchdog will unmount file systems and send
SIGTERM to all daemons. The duration of an extended failure is configurable
with watchdog_daemon_timeout.
ceph:
watchdog:
daemon_restart [default: no]: restart daemon if "normal" exit (status==0).
daemon_timeout [default: 300]: number of seconds a daemon
is allowed to be failed before the
watchdog will bark.
"""
def __init__(self, ctx, config, thrashers):
super(DaemonWatchdog, self).__init__()
self.config = ctx.config.get('watchdog', {})
self.ctx = ctx
self.e = None
self.logger = log.getChild('daemon_watchdog')
self.cluster = config.get('cluster', 'ceph')
self.name = 'watchdog'
self.stopping = Event()
self.thrashers = thrashers
def _run(self):
try:
self.watch()
except Exception as e:
# See _run exception comment for MDSThrasher
self.e = e
self.logger.exception("exception:")
# allow successful completion so gevent doesn't see an exception...
def log(self, x):
"""Write data to logger"""
self.logger.info(x)
def stop(self):
self.stopping.set()
def bark(self):
self.log("BARK! unmounting mounts and killing all daemons")
if hasattr(self.ctx, 'mounts'):
for mount in self.ctx.mounts.values():
try:
mount.umount_wait(force=True)
except:
self.logger.exception("ignoring exception:")
daemons = []
daemons.extend(filter(lambda daemon: daemon.running() and not daemon.proc.finished, self.ctx.daemons.iter_daemons_of_role('osd', cluster=self.cluster)))
daemons.extend(filter(lambda daemon: daemon.running() and not daemon.proc.finished, self.ctx.daemons.iter_daemons_of_role('mds', cluster=self.cluster)))
daemons.extend(filter(lambda daemon: daemon.running() and not daemon.proc.finished, self.ctx.daemons.iter_daemons_of_role('mon', cluster=self.cluster)))
daemons.extend(filter(lambda daemon: daemon.running() and not daemon.proc.finished, self.ctx.daemons.iter_daemons_of_role('rgw', cluster=self.cluster)))
daemons.extend(filter(lambda daemon: daemon.running() and not daemon.proc.finished, self.ctx.daemons.iter_daemons_of_role('mgr', cluster=self.cluster)))
for daemon in daemons:
try:
daemon.signal(signal.SIGTERM)
except:
self.logger.exception("ignoring exception:")
def watch(self):
self.log("watchdog starting")
daemon_timeout = int(self.config.get('daemon_timeout', 300))
daemon_restart = self.config.get('daemon_restart', False)
daemon_failure_time = {}
while not self.stopping.is_set():
bark = False
now = time.time()
osds = self.ctx.daemons.iter_daemons_of_role('osd', cluster=self.cluster)
mons = self.ctx.daemons.iter_daemons_of_role('mon', cluster=self.cluster)
mdss = self.ctx.daemons.iter_daemons_of_role('mds', cluster=self.cluster)
rgws = self.ctx.daemons.iter_daemons_of_role('rgw', cluster=self.cluster)
mgrs = self.ctx.daemons.iter_daemons_of_role('mgr', cluster=self.cluster)
daemon_failures = []
daemon_failures.extend(filter(lambda daemon: daemon.running() and daemon.proc.finished, osds))
daemon_failures.extend(filter(lambda daemon: daemon.running() and daemon.proc.finished, mons))
daemon_failures.extend(filter(lambda daemon: daemon.running() and daemon.proc.finished, mdss))
daemon_failures.extend(filter(lambda daemon: daemon.running() and daemon.proc.finished, rgws))
daemon_failures.extend(filter(lambda daemon: daemon.running() and daemon.proc.finished, mgrs))
for daemon in daemon_failures:
name = daemon.role + '.' + daemon.id_
dt = daemon_failure_time.setdefault(name, (daemon, now))
assert dt[0] is daemon
delta = now-dt[1]
self.log("daemon {name} is failed for ~{t:.0f}s".format(name=name, t=delta))
if delta > daemon_timeout:
bark = True
if daemon_restart == 'normal' and daemon.proc.exitstatus == 0:
self.log(f"attempting to restart daemon {name}")
daemon.restart()
# If a daemon is no longer failed, remove it from tracking:
for name in list(daemon_failure_time.keys()):
if name not in [d.role + '.' + d.id_ for d in daemon_failures]:
self.log("daemon {name} has been restored".format(name=name))
del daemon_failure_time[name]
for thrasher in self.thrashers:
if thrasher.exception is not None:
self.log("{name} failed".format(name=thrasher.name))
bark = True
if bark:
self.bark()
return
sleep(5)
self.log("watchdog finished")
|
