1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
|
import json
import os
import time
import re
from typing import Dict
from util import (
BoxType,
Config,
Target,
ensure_inside_container,
ensure_outside_container,
get_orch_hosts,
run_cephadm_shell_command,
run_dc_shell_command,
get_container_engine,
run_shell_command,
)
DEVICES_FILE="./devices.json"
def remove_loop_img() -> None:
loop_image = Config.get('loop_img')
if os.path.exists(loop_image):
os.remove(loop_image)
def create_loopback_devices(osds: int) -> Dict[int, Dict[str, str]]:
assert osds
cleanup_osds()
osd_devs = dict()
for i in range(osds):
img_name = f'osd{i}'
loop_dev = create_loopback_device(img_name)
osd_devs[i] = dict(img_name=img_name, device=loop_dev)
with open(DEVICES_FILE, 'w') as dev_file:
dev_file.write(json.dumps(osd_devs))
return osd_devs
def create_loopback_device(img_name, size_gb=5):
loop_img_dir = Config.get('loop_img_dir')
run_shell_command(f'mkdir -p {loop_img_dir}')
loop_img = os.path.join(loop_img_dir, img_name)
run_shell_command(f'rm -f {loop_img}')
run_shell_command(f'dd if=/dev/zero of={loop_img} bs=1 count=0 seek={size_gb}G')
loop_dev = run_shell_command(f'sudo losetup -f')
if not os.path.exists(loop_dev):
dev_minor = re.match(r'\/dev\/[^\d]+(\d+)', loop_dev).groups()[0]
run_shell_command(f'sudo mknod -m777 {loop_dev} b 7 {dev_minor}')
run_shell_command(f'sudo chown {os.getuid()}:{os.getgid()} {loop_dev}')
if os.path.ismount(loop_dev):
os.umount(loop_dev)
run_shell_command(f'sudo losetup {loop_dev} {loop_img}')
run_shell_command(f'sudo chown {os.getuid()}:{os.getgid()} {loop_dev}')
return loop_dev
def get_lvm_osd_data(data: str) -> Dict[str, str]:
osd_lvm_info = run_cephadm_shell_command(f'ceph-volume lvm list {data}')
osd_data = {}
for line in osd_lvm_info.split('\n'):
line = line.strip()
if not line:
continue
line = line.split()
if line[0].startswith('===') or line[0].startswith('[block]'):
continue
# "block device" key -> "block_device"
key = '_'.join(line[:-1])
osd_data[key] = line[-1]
return osd_data
def load_osd_devices():
if not os.path.exists(DEVICES_FILE):
return dict()
with open(DEVICES_FILE) as dev_file:
devs = json.loads(dev_file.read())
return devs
@ensure_inside_container
def deploy_osd(data: str, hostname: str) -> bool:
out = run_cephadm_shell_command(f'ceph orch daemon add osd {hostname}:{data} raw')
return 'Created osd(s)' in out
def cleanup_osds() -> None:
loop_img_dir = Config.get('loop_img_dir')
osd_devs = load_osd_devices()
for osd in osd_devs.values():
device = osd['device']
if 'loop' in device:
loop_img = os.path.join(loop_img_dir, osd['img_name'])
run_shell_command(f'sudo losetup -d {device}', expect_error=True)
if os.path.exists(loop_img):
os.remove(loop_img)
run_shell_command(f'rm -rf {loop_img_dir}')
def deploy_osds(count: int):
osd_devs = load_osd_devices()
hosts = get_orch_hosts()
host_index = 0
seed = get_container_engine().get_seed()
v = '-v' if Config.get('verbose') else ''
for osd in osd_devs.values():
deployed = False
while not deployed:
print(hosts)
hostname = hosts[host_index]['hostname']
deployed = run_dc_shell_command(
f'/cephadm/box/box.py {v} osd deploy --data {osd["device"]} --hostname {hostname}',
seed
)
deployed = 'created osd' in deployed.lower() or 'already created?' in deployed.lower()
print('Waiting 5 seconds to re-run deploy osd...')
time.sleep(5)
host_index = (host_index + 1) % len(hosts)
class Osd(Target):
_help = """
Deploy osds and create needed block devices with loopback devices:
Actions:
- deploy: Deploy an osd given a block device
- create_loop: Create needed loopback devices and block devices in logical volumes
for a number of osds.
- destroy: Remove all osds and the underlying loopback devices.
"""
actions = ['deploy', 'create_loop', 'destroy']
def set_args(self):
self.parser.add_argument('action', choices=Osd.actions)
self.parser.add_argument('--data', type=str, help='path to a block device')
self.parser.add_argument('--hostname', type=str, help='host to deploy osd')
self.parser.add_argument('--osds', type=int, default=0, help='number of osds')
def deploy(self):
data = Config.get('data')
hostname = Config.get('hostname')
if not hostname:
# assume this host
hostname = run_shell_command('hostname')
if not data:
deploy_osds(Config.get('osds'))
else:
deploy_osd(data, hostname)
@ensure_outside_container
def create_loop(self):
osds = Config.get('osds')
create_loopback_devices(int(osds))
print('Successfully created loopback devices')
@ensure_outside_container
def destroy(self):
cleanup_osds()
|
