1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
|
"""
Daemon restart
"""
import logging
import pipes
import os
from teuthology import misc as teuthology
from teuthology.orchestra import run as tor
from teuthology.orchestra import run
log = logging.getLogger(__name__)
def restart_daemon(ctx, config, role, id_, *args):
"""
Handle restart (including the execution of the command parameters passed)
"""
log.info('Restarting {r}.{i} daemon...'.format(r=role, i=id_))
daemon = ctx.daemons.get_daemon(role, id_)
log.debug('Waiting for exit of {r}.{i} daemon...'.format(r=role, i=id_))
try:
daemon.wait_for_exit()
except tor.CommandFailedError as e:
log.debug('Command Failed: {e}'.format(e=e))
if len(args) > 0:
confargs = ['--{k}={v}'.format(k=k, v=v) for k,v in zip(args[0::2], args[1::2])]
log.debug('Doing restart of {r}.{i} daemon with args: {a}...'.format(r=role, i=id_, a=confargs))
daemon.restart_with_args(confargs)
else:
log.debug('Doing restart of {r}.{i} daemon...'.format(r=role, i=id_))
daemon.restart()
def get_tests(ctx, config, role, remote, testdir):
"""Download restart tests"""
srcdir = '{tdir}/restart.{role}'.format(tdir=testdir, role=role)
refspec = config.get('branch')
if refspec is None:
refspec = config.get('sha1')
if refspec is None:
refspec = config.get('tag')
if refspec is None:
refspec = 'HEAD'
log.info('Pulling restart qa/workunits from ref %s', refspec)
remote.run(
logger=log.getChild(role),
args=[
'mkdir', '--', srcdir,
run.Raw('&&'),
'git',
'clone',
'https://git.ceph.com/ceph.git',
srcdir,
run.Raw('&&'),
'cd', '--', srcdir,
run.Raw('&&'),
'git', 'checkout', '-b', 'restart_test', str(refspec),
run.Raw('&&'),
'cd', '--', 'qa/workunits',
run.Raw('&&'),
'if', 'test', '-e', 'Makefile', run.Raw(';'), 'then', 'make', run.Raw(';'), 'fi',
run.Raw('&&'),
'find', '-executable', '-type', 'f', '-printf', r'%P\0',
run.Raw('>{tdir}/restarts.list'.format(tdir=testdir)),
],
)
restarts = sorted(remote.read_file(f'{testdir}/restarts.list').decode().split('\0'))
return (os.path.join(srcdir, 'qa/workunits'), restarts)
def task(ctx, config):
"""
Execute commands and allow daemon restart with config options.
Each process executed can output to stdout restart commands of the form:
restart <role> <id> <conf_key1> <conf_value1> <conf_key2> <conf_value2>
This will restart the daemon <role>.<id> with the specified config values once
by modifying the conf file with those values, and then replacing the old conf file
once the daemon is restarted.
This task does not kill a running daemon, it assumes the daemon will abort on an
assert specified in the config.
tasks:
- install:
- ceph:
- restart:
exec:
client.0:
- test_backtraces.py
"""
assert isinstance(config, dict), "task kill got invalid config"
testdir = teuthology.get_testdir(ctx)
try:
assert 'exec' in config, "config requires exec key with <role>: <command> entries"
for role, task in config['exec'].items():
log.info('restart for role {r}'.format(r=role))
(remote,) = ctx.cluster.only(role).remotes.keys()
srcdir, restarts = get_tests(ctx, config, role, remote, testdir)
log.info('Running command on role %s host %s', role, remote.name)
spec = '{spec}'.format(spec=task[0])
log.info('Restarts list: %s', restarts)
log.info('Spec is %s', spec)
to_run = [w for w in restarts if w == task or w.find(spec) != -1]
log.info('To run: %s', to_run)
for c in to_run:
log.info('Running restart script %s...', c)
args = [
run.Raw('TESTDIR="{tdir}"'.format(tdir=testdir)),
]
env = config.get('env')
if env is not None:
for var, val in env.items():
quoted_val = pipes.quote(val)
env_arg = '{var}={val}'.format(var=var, val=quoted_val)
args.append(run.Raw(env_arg))
args.extend([
'adjust-ulimits',
'ceph-coverage',
'{tdir}/archive/coverage'.format(tdir=testdir),
'{srcdir}/{c}'.format(
srcdir=srcdir,
c=c,
),
])
proc = remote.run(
args=args,
stdout=tor.PIPE,
stdin=tor.PIPE,
stderr=log,
wait=False,
)
log.info('waiting for a command from script')
while True:
l = proc.stdout.readline()
if not l or l == '':
break
log.debug('script command: {c}'.format(c=l))
ll = l.strip()
cmd = ll.split(' ')
if cmd[0] == "done":
break
assert cmd[0] == 'restart', "script sent invalid command request to kill task"
# cmd should be: restart <role> <id> <conf_key1> <conf_value1> <conf_key2> <conf_value2>
# or to clear, just: restart <role> <id>
restart_daemon(ctx, config, cmd[1], cmd[2], *cmd[3:])
proc.stdin.writelines(['restarted\n'])
proc.stdin.flush()
try:
proc.wait()
except tor.CommandFailedError:
raise Exception('restart task got non-zero exit status from script: {s}'.format(s=c))
finally:
log.info('Finishing %s on %s...', task, role)
remote.run(
logger=log.getChild(role),
args=[
'rm', '-rf', '--', '{tdir}/restarts.list'.format(tdir=testdir), srcdir,
],
)
|
